diff --git a/CHANGELOG.md b/CHANGELOG.md index fbae90b..f614c8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,34 @@ +## 6.0.0 + - SSL settings that were marked deprecated in version `5.6.0` are now marked obsolete, and will prevent the plugin from starting. + - These settings are: + - `cacert`, which should be replaced by `ssl_certificate_authorities` + - `client_cert`, which should be replaced by `ssl_certificate` + - `client_key`, which should be replaced by `ssl_key` + - `keystore`, which should be replaced by `ssl_keystore_path` + - `keystore_password`, which should be replaced by `ssl_keystore_password` + - `keystore_type`, which should be replaced by `ssl_keystore_password` + - `truststore`, which should be replaced by `ssl_truststore_path>` + - `truststore_password`, which should be replaced by `ssl_truststore_password` + - `truststore_type`, which should be replaced by `ssl_truststore_type` + - [#147](https://github.com/logstash-plugins/logstash-output-http/pull/147) + +## 5.7.1 + - Added new development `rackup` dependency to fix tests + +## 5.7.0 + - Added new `ssl_enabled` setting for enabling/disabling the SSL configurations [#144](https://github.com/logstash-plugins/logstash-output-http/pull/144) + +## 5.6.1 + - Added body logging for non 2xx responses [#142](https://github.com/logstash-plugins/logstash-output-http/pull/142) + +## 5.6.0 + - Added standardized SSL settings and deprecates their non-standard counterparts. Deprecated settings will continue to work, and will provide pipeline maintainers with guidance toward using their standardized counterparts [#140](https://github.com/logstash-plugins/logstash-output-http/pull/140) + - Added new `ssl_truststore_path`, `ssl_truststore_password`, and `ssl_truststore_type` settings for configuring SSL-trust using a PKCS-12 or JKS trust store, deprecating their `truststore`, `truststore_password`, and `truststore_type` counterparts. + - Added new `ssl_certificate_authorities` setting for configuring SSL-trust using a PEM-formatted list certificate authorities, deprecating its `cacert` counterpart. + - Added new `ssl_keystore_path`, `ssl_keystore_password`, and `ssl_keystore_type` settings for configuring SSL-identity using a PKCS-12 or JKS key store, deprecating their `keystore`, `keystore_password`, and `keystore_type` counterparts. + - Added new `ssl_certificate` and `ssl_key` settings for configuring SSL-identity using a PEM-formatted certificate/key pair, deprecating their `client_cert` and `client_key` counterparts. + - Added the `ssl_cipher_suites` option + ## 5.5.0 - Feat: added `ssl_supported_protocols` option [#131](https://github.com/logstash-plugins/logstash-output-http/pull/131) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index b61bbc5..78242ca 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -70,13 +70,13 @@ NOTE: The `retry_failed` option does not control the library level retry. This plugin supports the following configuration options plus the <> described later. +NOTE: As of version `6.0.0` of this plugin, a number of previously deprecated settings related to SSL have been removed. +Please check out <> for details. + [cols="<,<,<",options="header",] |======================================================================= |Setting |Input type|Required | <> |<>|No -| <> |a valid filesystem path|No -| <> |a valid filesystem path|No -| <> |a valid filesystem path|No | <> |<>|No | <> |<>|No | <> |<>|No @@ -87,9 +87,6 @@ This plugin supports the following configuration options plus the <> |<>, one of `["put", "post", "patch", "delete", "get", "head"]`|Yes | <> |<>|No | <> |<>|No -| <> |a valid filesystem path|No -| <> |<>|No -| <> |<>|No | <> |<>|No | <> |<>|No | <> |<>|No @@ -100,11 +97,18 @@ This plugin supports the following configuration options plus the <> |<>|No | <> |<>|No | <> |<>|No +| <> |<>|No +| <> |list of <>|No +| <> |list of <>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No | <> |<>|No -| <> |<>|No -| <> |a valid filesystem path|No -| <> |<>|No -| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>, one of `["full", "none"]`|No | <> |<>|Yes | <> |<>|No |======================================================================= @@ -125,29 +129,6 @@ to a value other than zero if the <> for more information. -[id="plugins-{type}s-{plugin}-cacert"] -===== `cacert` - - * Value type is <> - * There is no default value for this setting. - -If you need to use a custom X.509 CA (.pem certs) specify the path to that here - -[id="plugins-{type}s-{plugin}-client_cert"] -===== `client_cert` - - * Value type is <> - * There is no default value for this setting. - -If you'd like to use a client certificate (note, most people don't want this) set the path to the x509 cert here - -[id="plugins-{type}s-{plugin}-client_key"] -===== `client_key` - - * Value type is <> - * There is no default value for this setting. - -If you're using a client certificate specify the path to the encryption key here [id="plugins-{type}s-{plugin}-connect_timeout"] ===== `connect_timeout` @@ -252,31 +233,6 @@ enumerate them here. Responses returning these codes will be considered successe Turn this on to enable HTTP keepalive support. We highly recommend setting `automatic_retries` to at least one with this to fix interactions with broken keepalive implementations. -[id="plugins-{type}s-{plugin}-keystore"] -===== `keystore` - - * Value type is <> - * There is no default value for this setting. - -If you need to use a custom keystore (`.jks`) specify that here. This does not work with .pem keys! - -[id="plugins-{type}s-{plugin}-keystore_password"] -===== `keystore_password` - - * Value type is <> - * There is no default value for this setting. - -Specify the keystore password here. -Note, most .jks files created with keytool require a password! - -[id="plugins-{type}s-{plugin}-keystore_type"] -===== `keystore_type` - - * Value type is <> - * Default value is `"JKS"` - -Specify the keystore type here. One of `JKS` or `PKCS12`. Default is `JKS` - [id="plugins-{type}s-{plugin}-mapping"] ===== `mapping` @@ -379,6 +335,79 @@ See <> for more information. Timeout (in seconds) to wait for data on the socket. Default is `10s` +[id="plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is <> + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <> is set. + +[id="plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of <> + * This plugin does not support multiple entries + * There is no default value for this setting + +The .cer or .pem CA files to validate the server's certificate. + +NOTE: When provided with a single PEM file that contains multiple certificates, all enclosed certificates will be trusted. + +[id="plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is a list of <> + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is <> + * Default value is `true` + +Enable SSL/TLS secured communication. It must be `true` for other `ssl_` options +to take effect. + +[id="plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is <> + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the <>. + +NOTE: This setting can be used only if <> is set. + +[id="plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is <> + * There is no default value for this setting. + +Set the keystore password + +[id="plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is <> + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +[id="plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + [id="plugins-{type}s-{plugin}-ssl_supported_protocols"] ===== `ssl_supported_protocols` @@ -396,6 +425,31 @@ NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. +[id="plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is <> + * There is no default value for this setting. + +Set the truststore password + +[id="plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is <> + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +[id="plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + [id="plugins-{type}s-{plugin}-ssl_verification_mode"] ===== `ssl_verification_mode` @@ -412,31 +466,6 @@ This mode disables many of the security benefits of SSL/TLS and should only be u It is primarily intended as a temporary diagnostic mechanism when attempting to resolve TLS errors. Using `none` in production environments is strongly discouraged. -[id="plugins-{type}s-{plugin}-truststore"] -===== `truststore` - - * Value type is <> - * There is no default value for this setting. - -If you need to use a custom truststore (`.jks`) specify that here. This does not work with .pem certs! - -[id="plugins-{type}s-{plugin}-truststore_password"] -===== `truststore_password` - - * Value type is <> - * There is no default value for this setting. - -Specify the truststore password here. -Note, most .jks files created with keytool require a password! - -[id="plugins-{type}s-{plugin}-truststore_type"] -===== `truststore_type` - - * Value type is <> - * Default value is `"JKS"` - -Specify the truststore type here. One of `JKS` or `PKCS12`. Default is `JKS` - [id="plugins-{type}s-{plugin}-url"] ===== `url` @@ -459,8 +488,29 @@ Quoting the Apache commons docs (this client is based Apache Commmons): See https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[these docs for more info] +[id="plugins-{type}s-{plugin}-obsolete-options"] +==== HTTP Output Obsolete Configuration Options + +WARNING: As of version `6.0.0` of this plugin, some configuration options have been replaced. +The plugin will fail to start if it contains any of these obsolete options. + + +[cols="<,<",options="header",] +|======================================================================= +|Setting|Replaced by +| cacert |<> +| client_cert |<> +| client_key |<> +| keystore |<> +| keystore_password |<> +| keystore_type |<> +| truststore |<> +| truststore_password |<> +| truststore_type |<> +|======================================================================= + [id="plugins-{type}s-{plugin}-common-options"] include::{include_path}/{type}.asciidoc[] -:default_codec!: \ No newline at end of file +:default_codec!: diff --git a/lib/logstash/outputs/http.rb b/lib/logstash/outputs/http.rb index 3f9bac1..5d81698 100644 --- a/lib/logstash/outputs/http.rb +++ b/lib/logstash/outputs/http.rb @@ -7,7 +7,7 @@ require "zlib" class LogStash::Outputs::Http < LogStash::Outputs::Base - include LogStash::PluginMixins::HttpClient + include LogStash::PluginMixins::HttpClient[:with_obsolete => true] concurrency :shared @@ -154,12 +154,12 @@ def log_retryable_response(response) end def log_error_response(response, url, event) - log_failure( - "Encountered non-2xx HTTP code #{response.code}", - :response_code => response.code, - :url => url, - :event => event - ) + log_details = { + :code => response.code, + :body => response.body, + :url => url, + } + log_failure("Encountered non-2xx HTTP code", log_details) end def send_events(events) diff --git a/logstash-output-http.gemspec b/logstash-output-http.gemspec index 56128f0..d10ce6d 100644 --- a/logstash-output-http.gemspec +++ b/logstash-output-http.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = 'logstash-output-http' - s.version = '5.5.0' + s.version = '6.0.0' s.licenses = ['Apache License (2.0)'] s.summary = "Sends events to a generic HTTP or HTTPS endpoint" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" @@ -20,9 +20,12 @@ Gem::Specification.new do |s| # Gem dependencies s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99" - s.add_runtime_dependency "logstash-mixin-http_client", ">= 7.2.0", "< 8.0.0" + s.add_runtime_dependency "logstash-mixin-http_client", ">= 7.5.0", "< 8.0.0" s.add_development_dependency 'logstash-devutils' s.add_development_dependency 'sinatra' s.add_development_dependency 'webrick' + + # Pin to avoid using new Fiber-based implementation that breaks tests here + s.add_development_dependency 'rackup', "< 2.1.0" end diff --git a/spec/outputs/http_spec.rb b/spec/outputs/http_spec.rb index 26d2f59..adceccd 100644 --- a/spec/outputs/http_spec.rb +++ b/spec/outputs/http_spec.rb @@ -20,6 +20,32 @@ let(:url) { "http://localhost:#{port}/good" } let(:method) { "post" } + describe "obsolete settings" do + let(:config) { {"url" => url, "http_method" => "post"} } + + [{:name => 'cacert', :canonical_name => 'ssl_certificate_authorities'}, + {:name => 'client_cert', :canonical_name => 'ssl_certificate'}, + {:name => 'client_key', :canonical_name => 'ssl_key'}, + {:name => "keystore", :canonical_name => 'ssl_keystore_path'}, + {:name => 'truststore', :canonical_name => 'ssl_truststore_path'}, + {:name => "keystore_password", :canonical_name => "ssl_keystore_password"}, + {:name => 'truststore_password', :canonical_name => "ssl_truststore_password"}, + {:name => "keystore_type", :canonical_name => "ssl_keystore_type"}, + {:name => 'truststore_type', :canonical_name => 'ssl_truststore_type'} + ].each do |settings| + context "with option #{settings[:name]}" do + let(:obsolete_config) { config.merge(settings[:name] => 'test_value') } + + it "emits an error about the setting `#{settings[:name]}` now being obsolete and provides guidance to use `#{settings[:canonical_name]}`" do + error_text = /The setting `#{settings[:name]}` in plugin `http` is obsolete and is no longer available. Use `#{settings[:canonical_name]}` instead/i + expect { LogStash::Outputs::Http.new(obsolete_config) }.to raise_error LogStash::ConfigurationError, error_text + end + + end + end + end + + shared_examples("verb behavior") do |method| shared_examples("failure log behaviour") do @@ -255,7 +281,7 @@ end let(:last_request) { TestApp.last_request } - let(:body) { last_request.body.read } + let(:body) { read_last_request_body(last_request) } let(:content_type) { last_request.env["CONTENT_TYPE"] } it "should receive the request" do @@ -304,7 +330,7 @@ let(:config) { base_config.merge({"url" => url, "http_method" => "post", "pool_max" => 1}) } - let(:expected_body) { LogStash::Json.dump(event) } + let(:expected_body) { event.to_json } let(:expected_content_type) { "application/json" } include_examples("a received event") @@ -417,10 +443,10 @@ RSpec.describe LogStash::Outputs::Http do # different block as we're starting web server with TLS - @@default_server_settings = TestApp.server_settings.dup + let(:default_server_settings) { TestApp.server_settings.dup } before do - TestApp.server_settings = @@default_server_settings.merge(webrick_config) + TestApp.server_settings = default_server_settings.merge(webrick_config) TestApp.last_request = nil @@ -441,7 +467,7 @@ @server.shutdown # WEBrick::HTTPServer TestApp.stop! rescue nil - TestApp.server_settings = @@default_server_settings + TestApp.server_settings = default_server_settings end let(:ssl_cert_host) { 'localhost' } @@ -458,7 +484,7 @@ after { subject.close } let(:last_request) { TestApp.last_request } - let(:last_request_body) { last_request.body.read } + let(:last_request_body) { read_last_request_body(last_request) } let(:event) { LogStash::Event.new("message" => "hello!") } @@ -530,3 +556,10 @@ end if tls_version_enabled_by_default?('TLSv1.3') && JOpenSSL::VERSION > '0.12' # due WEBrick uses OpenSSL end + +# Pre-emptively rewind the retrieval of `last_request.body` - for form based endpoints, the body +# is placed in params, and body is empty, requiring a `rewind` for the body to be available for comparison +def read_last_request_body(last_request) + last_request.body.rewind + last_request.body.read +end \ No newline at end of file