From 1c99fa8fb4c66da2a434108579675b076900feab Mon Sep 17 00:00:00 2001 From: edmocosta <11836452+edmocosta@users.noreply.github.com> Date: Thu, 2 Mar 2023 18:56:35 +0100 Subject: [PATCH 01/10] Deprecated and added new SSL settings --- CHANGELOG.md | 18 ++ docs/index.asciidoc | 162 ++++++++++++- lib/logstash/inputs/elasticsearch.rb | 165 +++++++++++-- logstash-input-elasticsearch.gemspec | 3 +- spec/inputs/elasticsearch_spec.rb | 6 +- spec/inputs/elasticsearch_ssl_spec.rb | 216 ++++++++++++++++++ spec/inputs/integration/elasticsearch_spec.rb | 8 +- 7 files changed, 549 insertions(+), 29 deletions(-) create mode 100644 spec/inputs/elasticsearch_ssl_spec.rb diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a973686..f9e0b8e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,21 @@ +## 4.17.0 +- Added SSL settings for: [#1115](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1115) + - `ssl_enabled`: Enable/disable the SSL settings. If not provided, the value is inferred from the hosts scheme + - `ssl_certificate`: OpenSSL-style X.509 certificate file to authenticate the client + - `ssl_key`: OpenSSL-style RSA private key that corresponds to the `ssl_certificate` + - `ssl_truststore_path`: he JKS truststore to validate the server's certificate + - `ssl_truststore_type`: The format of the truststore file + - `ssl_truststore_password`: The truststore password + - `ssl_keystore_path`: The keystore used to present a certificate to the server + - `ssl_keystore_type`: The format of the keystore file + - `ssl_keystore_password`: The keystore password + - `ssl_cipher_suites`: The list of cipher suites to use + - `ssl_supported_protocols`: Supported protocols with versions +- Reviewed and deprecated SSL settings to comply with Logstash's naming convention + - Deprecated `ssl` in favor of `ssl_enabled` + - Deprecated `ca_file` in favor of `ssl_certificate_authorities` + - Deprecated `ssl_certificate_verification` in favor of `ssl_verification_mode` + ## 4.16.0 - Added `ssl_certificate_verification` option to control SSL certificate verification [#180](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/180) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index d883c658..58715616 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -102,7 +102,7 @@ This plugin supports the following configuration options plus the <> |<>|No -| <> |a valid filesystem path|No +| <> |a valid filesystem path|__Deprecated__ | <> |<>|No | <> |<>|No | <> |<>|No @@ -121,8 +121,21 @@ This plugin supports the following configuration options plus the <> |<>|No | <> |<>|No | <> |<>|No -| <> |<>|No -| <> |<>|No +| <> |<>|__Deprecated__ +| <> |<>|No +| <> |list of <>|No +| <> |<>|__Deprecated__ +| <> |list of <>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>, one of `["full", "none"]`|No | <> | <>|No | <> | {logstash-ref}/field-references-deepdive.html[field reference] | No | <> | <>|No @@ -140,7 +153,7 @@ input plugins. * Value type is <> * There is no default value for this setting. -Authenticate using Elasticsearch API key. Note that this option also requires enabling the `ssl` option. +Authenticate using Elasticsearch API key. Note that this option also requires enabling the `ssl_enabled` option. Format is `id:api_key` where `id` and `api_key` are as returned by the Elasticsearch @@ -149,6 +162,7 @@ API key API]. [id="plugins-{type}s-{plugin}-ca_file"] ===== `ca_file` +deprecated[8.8.0, Replaced by <>] * Value type is <> * There is no default value for this setting. @@ -408,6 +422,7 @@ instructions into the query. [id="plugins-{type}s-{plugin}-ssl"] ===== `ssl` +deprecated[8.8.0, Replaced by <>] * Value type is <> * Default value is `false` @@ -415,11 +430,31 @@ instructions into the query. If enabled, SSL will be used when communicating with the Elasticsearch server (i.e. HTTPS will be used instead of plain HTTP). +[id="plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is <> + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if `ssl_key` is set. + +[id="plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of <> + * There is no default value for this setting + +The .cer or .pem files to validate the server's certificate. + +NOTE: You cannot use this setting and `ssl_truststore_path` at the same time. + [id="plugins-{type}s-{plugin}-ssl_certificate_verification"] ===== `ssl_certificate_verification` +deprecated[8.8.0, Replaced by <>] -* Value type is <> -* Default value is `true` + * Value type is <> + * Default value is `true` Option to validate the server's certificate. Disabling this severely compromises security. When certificate validation is disabled, this plugin implicitly trusts the machine @@ -429,6 +464,121 @@ man-in-the-middle or other compromised infrastructure. More information on the importance of certificate verification: **https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf**. +[id="plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of <> + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + + +[id="plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is <> + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <> or extracted from the <>. +If no explicit protocol is specified plain HTTP will be used. + +[id="plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is <> + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the `ssl_certificate`. + +NOTE: This setting can be used only if `ssl_certificate` is set. + +[id="plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is <> + * There is no default value for this setting. + +Set the keystore password + +[id="plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is <> + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either .jks or .p12 + +NOTE: You cannot use this setting and `ssl_certificate` at the same time. + +[id="plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is <> + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is <> + * There is no default value for this setting. + +Set the truststore password + +[id="plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is <> + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either .jks or .p12. + +NOTE: You cannot use this setting and `ssl_certificate_authorities` at the same time. + +[id="plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + [id="plugins-{type}s-{plugin}-socket_timeout_seconds"] ===== `socket_timeout_seconds` diff --git a/lib/logstash/inputs/elasticsearch.rb b/lib/logstash/inputs/elasticsearch.rb index 7edcf727..b18bc2b4 100644 --- a/lib/logstash/inputs/elasticsearch.rb +++ b/lib/logstash/inputs/elasticsearch.rb @@ -9,6 +9,7 @@ require 'logstash/plugin_mixins/ecs_compatibility_support/target_check' require 'logstash/plugin_mixins/ca_trusted_fingerprint_support' require "logstash/plugin_mixins/scheduler" +require "logstash/plugin_mixins/normalize_config_support" require "base64" require 'logstash/helpers/loggable_try' @@ -82,6 +83,8 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base include LogStash::PluginMixins::Scheduler + include LogStash::PluginMixins::NormalizeConfigSupport + config_name "elasticsearch" # List of elasticsearch hosts to use for querying. @@ -185,15 +188,60 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base config :proxy, :validate => :uri_or_empty # SSL - config :ssl, :validate => :boolean, :default => false + config :ssl, :validate => :boolean, :default => false, :deprecated => "Set 'ssl_enabled' instead." + + # SSL + config :ssl_enabled, :validate => :boolean # SSL Certificate Authority file in PEM encoded format, must also include any chain certificates as necessary - config :ca_file, :validate => :path + config :ca_file, :validate => :path, :deprecated => "Set 'ssl_certificate_authorities' instead." + + # SSL Certificate Authority files in PEM encoded format, must also include any chain certificates as necessary + config :ssl_certificate_authorities, :validate => :path, :list => true # Option to validate the server's certificate. Disabling this severely compromises security. # For more information on the importance of certificate verification please read # https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf - config :ssl_certificate_verification, :validate => :boolean, :default => true + config :ssl_certificate_verification, :validate => :boolean, :default => true, :deprecated => "Set 'ssl_verification_mode' instead." + + # Options to verify the server's certificate. + # "full": validates that the provided certificate has an issue date that’s within the not_before and not_after dates; + # chains to a trusted Certificate Authority (CA); has a hostname or IP address that matches the names within the certificate. + # "none": performs no certificate validation. Disabling this severely compromises security (https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf) + config :ssl_verification_mode, :validate => %w[full none], :default => 'full' + + # OpenSSL-style X.509 certificate certificate to authenticate the client + config :ssl_certificate, :validate => :path + + # OpenSSL-style RSA private key to authenticate the client + config :ssl_key, :validate => :path + + # The JKS truststore to validate the server's certificate. + # Use either `:ssl_truststore_path` or `:ssl_certificate_authorities` + config :ssl_truststore_path, :validate => :path + + # The format of the truststore file. It must be either jks or pkcs12 + config :ssl_truststore_type, :validate => %w[pkcs12 jks] + + # Set the truststore password + config :ssl_truststore_password, :validate => :password + + # The keystore used to present a certificate to the server. + # It can be either .jks or .p12 + config :ssl_keystore_path, :validate => :path + + # The format of the keystore file. It must be either jks or pkcs12 + config :ssl_keystore_type, :validate => %w[pkcs12 jks] + + # Set the keystore password + config :ssl_keystore_password, :validate => :password + + # The list of cipher suites to use, listed by priorities. + # Supported cipher suites vary depending on which version of Java is used. + config :ssl_cipher_suites, :validate => :string, :list => true + + # Supported protocols with versions. + config :ssl_supported_protocols, :validate => %w[TLSv1.1 TLSv1.2 TLSv1.3], :default => [], :list => true # Schedule of when to periodically run statement, in Cron format # for example: "* * * * *" (execute query every minute, on the minute) @@ -211,6 +259,8 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base def initialize(params={}) super(params) + setup_ssl_params + if docinfo_target.nil? @docinfo_target = ecs_select[disabled: '@metadata', v1: '[@metadata][input][elasticsearch]'] end @@ -246,7 +296,7 @@ def register transport_options[:socket_timeout] = @socket_timeout_seconds unless @socket_timeout_seconds.nil? hosts = setup_hosts - ssl_options = setup_ssl + ssl_options = setup_client_ssl @logger.warn "Supplied proxy setting (proxy => '') has no effect" if @proxy.eql?('') @@ -416,6 +466,15 @@ def hosts_default?(hosts) hosts.nil? || ( hosts.is_a?(Array) && hosts.empty? ) end + def effectively_ssl? + return @ssl_enabled unless @ssl_enabled.nil? + + hosts = Array(@hosts) + return false if hosts.nil? || hosts.empty? + + hosts.all? { |host| host && host.start_with?("https") } + end + def validate_authentication authn_options = 0 authn_options += 1 if @cloud_auth @@ -426,26 +485,102 @@ def validate_authentication raise LogStash::ConfigurationError, 'Multiple authentication options are specified, please only use one of user/password, cloud_auth or api_key' end - if @api_key && @api_key.value && @ssl != true - raise(LogStash::ConfigurationError, "Using api_key authentication requires SSL/TLS secured communication using the `ssl => true` option") + if @api_key && @api_key.value && @ssl_enabled != true + raise(LogStash::ConfigurationError, "Using api_key authentication requires SSL/TLS secured communication using the `ssl_enabled => true` option") end end - def setup_ssl + def setup_client_ssl ssl_options = {} - - ssl_options[:ssl] = true if @ssl - ssl_options[:ca_file] = @ca_file if @ssl && @ca_file + ssl_options[:ssl] = true if @ssl_enabled ssl_options[:trust_strategy] = trust_strategy_for_ca_trusted_fingerprint - if @ssl && !@ssl_certificate_verification - logger.warn "You have enabled encryption but DISABLED certificate verification, " + - "to make sure your data is secure remove `ssl_certificate_verification => false`" - ssl_options[:verify] = :disable + + return ssl_options unless @ssl_enabled + + ssl_certificate_authorities, ssl_truststore_path, ssl_certificate, ssl_keystore_path = params.values_at('ssl_certificate_authorities', 'ssl_truststore_path', 'ssl_certificate', 'ssl_keystore_path') + + if ssl_certificate_authorities && ssl_truststore_path + raise(LogStash::ConfigurationError, 'Use either "ssl_certificate_authorities/ca_file" or "ssl_truststore_path" when configuring the CA certificate') + end + + if ssl_certificate && ssl_keystore_path + raise(LogStash::ConfigurationError, 'Use either "ssl_certificate" or "ssl_keystore_path/keystore" when configuring client certificates') + end + + if ssl_certificate_authorities&.any? + raise(LogStash::ConfigurationError, 'Multiple values on "ssl_certificate_authorities" are not supported by this plugin') if ssl_certificate_authorities.size > 1 + ssl_options[:ca_file] = ssl_certificate_authorities.first + end + + if ssl_truststore_path + ssl_options[:truststore] = ssl_truststore_path + ssl_options[:truststore_type] = params["ssl_truststore_type"] if params.include?("ssl_truststore_type") + ssl_options[:truststore_password] = params["ssl_truststore_password"].value if params.include?("ssl_truststore_password") + end + + if ssl_keystore_path + ssl_options[:keystore] = ssl_keystore_path + ssl_options[:keystore_type] = params["ssl_keystore_type"] if params.include?("ssl_keystore_type") + ssl_options[:keystore_password] = params["ssl_keystore_password"].value if params.include?("ssl_keystore_password") end + ssl_key = params["ssl_key"] + if ssl_certificate && ssl_key + ssl_options[:client_cert] = ssl_certificate + ssl_options[:client_key] = ssl_key + elsif !!ssl_certificate ^ !!ssl_key + raise(LogStash::ConfigurationError, 'You must set both "ssl_certificate" and "ssl_key" for client authentication') + end + + ssl_verification_mode = params["ssl_verification_mode"] + unless ssl_verification_mode.nil? + case ssl_verification_mode + when 'none' + logger.warn "You have enabled encryption but DISABLED certificate verification, " + + "to make sure your data is secure set `ssl_verification_mode => full`" + ssl_options[:verify] = :disable + else + ssl_options[:verify] = :strict + end + end + + ssl_options[:cipher_suites] = params["ssl_cipher_suites"] if params.include?("ssl_cipher_suites") + + protocols = params['ssl_supported_protocols'] + ssl_options[:protocols] = protocols if protocols&.any? + ssl_options end + def setup_ssl_params + @ssl_enabled = normalize_config(:ssl_enabled) do |normalize| + normalize.with_deprecated_alias(:ssl) + end + + # Infer the value if neither the deprecate `ssl` and `ssl_enabled` were set + @ssl_enabled = effectively_ssl? if @ssl_enabled.nil? + + @ssl_certificate_authorities = normalize_config(:ssl_certificate_authorities) do |normalize| + normalize.with_deprecated_mapping(:ca_file) do |ca_file| + [ca_file] + end + end + + @ssl_verification_mode = normalize_config(:ssl_verification_mode) do |normalize| + normalize.with_deprecated_mapping(:ssl_certificate_verification) do |ssl_certificate_verification| + if ssl_certificate_verification == true + "full" + else + "none" + end + end + end + + params['ssl_enabled'] = @ssl_enabled unless @ssl_enabled.nil? + params['ssl_certificate_authorities'] = @ssl_certificate_authorities unless @ssl_certificate_authorities.nil? + params['ssl_verification_mode'] = @ssl_verification_mode unless @ssl_verification_mode.nil? + end + def setup_hosts @hosts = Array(@hosts).map { |host| host.to_s } # potential SafeURI#to_s @hosts.map do |h| @@ -453,7 +588,7 @@ def setup_hosts h else host, port = h.split(':') - { host: host, port: port, scheme: (@ssl ? 'https' : 'http') } + { host: host, port: port, scheme: (@ssl_enabled ? 'https' : 'http') } end end end diff --git a/logstash-input-elasticsearch.gemspec b/logstash-input-elasticsearch.gemspec index b7aeefb0..5a98d2f0 100644 --- a/logstash-input-elasticsearch.gemspec +++ b/logstash-input-elasticsearch.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-elasticsearch' - s.version = '4.16.0' + s.version = '4.17.0' s.licenses = ['Apache License (2.0)'] s.summary = "Reads query results from an Elasticsearch cluster" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" @@ -28,6 +28,7 @@ Gem::Specification.new do |s| s.add_runtime_dependency 'elasticsearch', '>= 7.17.1' s.add_runtime_dependency 'logstash-mixin-ca_trusted_fingerprint_support', '~> 1.0' + s.add_runtime_dependency 'logstash-mixin-normalize_config_support', '~>1.0' s.add_runtime_dependency 'tzinfo' s.add_runtime_dependency 'tzinfo-data' diff --git a/spec/inputs/elasticsearch_spec.rb b/spec/inputs/elasticsearch_spec.rb index 8004bd83..1aa31410 100644 --- a/spec/inputs/elasticsearch_spec.rb +++ b/spec/inputs/elasticsearch_spec.rb @@ -681,7 +681,7 @@ def synchronize_method!(object, method_name) end context "with ssl" do - let(:config) { super().merge({ 'api_key' => LogStash::Util::Password.new('foo:bar'), "ssl" => true }) } + let(:config) { super().merge({ 'api_key' => LogStash::Util::Password.new('foo:bar'), "ssl_enabled" => true }) } it "should set authorization" do plugin.register @@ -700,9 +700,9 @@ def synchronize_method!(object, method_name) end context 'ssl verification disabled' do - let(:config) { super().merge({ 'ssl_certificate_verification' => false }) } + let(:config) { super().merge({ 'ssl_verification_mode' => 'none' }) } it 'should warn data security risk' do - expect(plugin.logger).to receive(:warn).once.with("You have enabled encryption but DISABLED certificate verification, to make sure your data is secure remove `ssl_certificate_verification => false`") + expect(plugin.logger).to receive(:warn).once.with("You have enabled encryption but DISABLED certificate verification, to make sure your data is secure set `ssl_verification_mode => full`") plugin.register end end diff --git a/spec/inputs/elasticsearch_ssl_spec.rb b/spec/inputs/elasticsearch_ssl_spec.rb new file mode 100644 index 00000000..96fd70b0 --- /dev/null +++ b/spec/inputs/elasticsearch_ssl_spec.rb @@ -0,0 +1,216 @@ +require 'stud/temporary' +require "elasticsearch" + +describe "SSL options" do + let(:es_client_double) { double("Elasticsearch::Client #{self.inspect}") } + let(:hosts) {["localhost"]} + let(:settings) { { "ssl_enabled" => true, "hosts" => hosts } } + + subject do + require "logstash/inputs/elasticsearch" + LogStash::Inputs::Elasticsearch.new(settings) + end + + before do + allow(es_client_double).to receive(:close) + allow(es_client_double).to receive(:ping).with(any_args).and_return(double("pong").as_null_object) + allow(Elasticsearch::Client).to receive(:new).and_return(es_client_double) + end + + after do + subject.close + end + + context "when neither ssl nor ssl_enabled is set" do + let(:settings) { super().reject { |k| %w[ssl ssl_enabled].include?(k) } } + + context "and there is no https hosts" do + let(:hosts) { %w[http://es01 http://es01] } + + it "should infer the ssl_enabled value to false" do + subject.register + expect(subject.instance_variable_get(:@ssl_enabled)).to eql(false) + expect(subject.params).to match hash_including("ssl_enabled" => false) + end + end + + context "and there is https hosts" do + let(:hosts) { %w[https://sec-es01 https://sec-es01] } + + it "should infer the ssl_enabled value to true" do + subject.register + expect(subject.instance_variable_get(:@ssl_enabled)).to eql(true) + expect(subject.params).to match hash_including("ssl_enabled" => true) + end + end + end + + context "when ssl_verification_mode" do + context "is set to none" do + let(:settings) { super().merge( + "ssl_verification_mode" => "none", + ) } + + it "should print a warning" do + expect(subject.logger).to receive(:warn).with(/You have enabled encryption but DISABLED certificate verification/).at_least(:once) + allow(subject.logger).to receive(:warn).with(any_args) + + subject.register + end + + it "should pass the flag to the ES client" do + expect(::Elasticsearch::Client).to receive(:new) do |args| + expect(args[:ssl]).to match hash_including(:ssl => true, :verify => :disable) + end.and_return(es_client_double) + + subject.register + end + end + + context "is set to full" do + let(:settings) { super().merge( + "ssl_verification_mode" => 'full', + ) } + + it "should pass the flag to the ES client" do + expect(::Elasticsearch::Client).to receive(:new) do |args| + expect(args[:ssl]).to match hash_including(:ssl => true, :verify => :strict) + end.and_return(es_client_double) + + subject.register + end + end + end + + context "with the conflicting configs" do + context "ssl_certificate_authorities and ssl_truststore_path set" do + let(:ssl_truststore_path) { Stud::Temporary.file.path } + let(:ssl_certificate_authorities_path) { Stud::Temporary.file.path } + let(:settings) { super().merge( + "ssl_truststore_path" => ssl_truststore_path, + "ssl_certificate_authorities" => ssl_certificate_authorities_path + ) } + + after :each do + File.delete(ssl_truststore_path) + File.delete(ssl_certificate_authorities_path) + end + + it "should raise a configuration error" do + expect { subject.register }.to raise_error(LogStash::ConfigurationError, /Use either "ssl_certificate_authorities\/ca_file" or "ssl_truststore_path"/) + end + end + + context "ssl_certificate and ssl_keystore_path set" do + let(:ssl_keystore_path) { Stud::Temporary.file.path } + let(:ssl_certificate_path) { Stud::Temporary.file.path } + let(:settings) { super().merge( + "ssl_certificate" => ssl_certificate_path, + "ssl_keystore_path" => ssl_keystore_path + ) } + + after :each do + File.delete(ssl_keystore_path) + File.delete(ssl_certificate_path) + end + + it "should raise a configuration error" do + expect { subject.register }.to raise_error(LogStash::ConfigurationError, /Use either "ssl_certificate" or "ssl_keystore_path\/keystore"/) + end + end + end + + context "when configured with Java store files" do + let(:ssl_truststore_path) { Stud::Temporary.file.path } + let(:ssl_keystore_path) { Stud::Temporary.file.path } + + after :each do + File.delete(ssl_truststore_path) + File.delete(ssl_keystore_path) + end + + let(:settings) { super().merge( + "ssl_truststore_path" => ssl_truststore_path, + "ssl_truststore_type" => "jks", + "ssl_truststore_password" => "foo", + "ssl_keystore_path" => ssl_keystore_path, + "ssl_keystore_type" => "jks", + "ssl_keystore_password" => "bar", + "ssl_verification_mode" => "full", + "ssl_cipher_suites" => ["TLS_DHE_RSA_WITH_AES_256_CBC_SHA256"], + "ssl_supported_protocols" => ["TLSv1.3"] + ) } + + it "should pass the parameters to the ES client" do + expect(::Elasticsearch::Client).to receive(:new) do |args| + expect(args[:ssl]).to match hash_including( + :ssl => true, + :keystore => ssl_keystore_path, + :keystore_type => "jks", + :keystore_password => "bar", + :truststore => ssl_truststore_path, + :truststore_type => "jks", + :truststore_password => "foo", + :verify => :strict, + :cipher_suites => ["TLS_DHE_RSA_WITH_AES_256_CBC_SHA256"], + :protocols => ["TLSv1.3"], + ) + end.and_return(es_client_double) + + subject.register + end + end + + context "when configured with certificate files" do + let(:ssl_certificate_authorities_path) { Stud::Temporary.file.path } + let(:ssl_certificate_path) { Stud::Temporary.file.path } + let(:ssl_key_path) { Stud::Temporary.file.path } + let(:settings) { super().merge( + "ssl_certificate_authorities" => [ssl_certificate_authorities_path], + "ssl_certificate" => ssl_certificate_path, + "ssl_key" => ssl_key_path, + "ssl_verification_mode" => "full", + "ssl_cipher_suites" => ["TLS_DHE_RSA_WITH_AES_256_CBC_SHA256"], + "ssl_supported_protocols" => ["TLSv1.3"] + ) } + + after :each do + File.delete(ssl_certificate_authorities_path) + File.delete(ssl_certificate_path) + File.delete(ssl_key_path) + end + + it "should pass the parameters to the ES client" do + expect(::Elasticsearch::Client).to receive(:new) do |args| + expect(args[:ssl]).to match hash_including( + :ssl => true, + :ca_file => ssl_certificate_authorities_path, + :client_cert => ssl_certificate_path, + :client_key => ssl_key_path, + :verify => :strict, + :cipher_suites => ["TLS_DHE_RSA_WITH_AES_256_CBC_SHA256"], + :protocols => ["TLSv1.3"], + ) + end.and_return(es_client_double) + + subject.register + end + + context "and only the ssl_certificate is set" do + let(:settings) { super().reject { |k| "ssl_key".eql?(k) } } + + it "should raise a configuration error" do + expect { subject.register }.to raise_error(LogStash::ConfigurationError, /You must set both "ssl_certificate" and "ssl_key"/) + end + end + + context "and only the ssl_key is set" do + let(:settings) { super().reject { |k| "ssl_certificate".eql?(k) } } + + it "should raise a configuration error" do + expect { subject.register }.to raise_error(LogStash::ConfigurationError, /You must set both "ssl_certificate" and "ssl_key"/) + end + end + end +end + diff --git a/spec/inputs/integration/elasticsearch_spec.rb b/spec/inputs/integration/elasticsearch_spec.rb index 0b9aee02..e9c02fa9 100644 --- a/spec/inputs/integration/elasticsearch_spec.rb +++ b/spec/inputs/integration/elasticsearch_spec.rb @@ -95,13 +95,13 @@ end context 'with ca_file' do - let(:config) { super().merge('ssl' => true, 'ca_file' => ca_file) } + let(:config) { super().merge('ssl_enabled' => true, 'ssl_certificate_authorities' => ca_file) } it_behaves_like 'secured_elasticsearch' end context 'with `ca_trusted_fingerprint`' do let(:ca_trusted_fingerprint) { File.read("spec/fixtures/test_certs/ca.der.sha256").chomp } - let(:config) { super().merge('ssl' => true, 'ca_trusted_fingerprint' => ca_trusted_fingerprint) } + let(:config) { super().merge('ssl_enabled' => true, 'ca_trusted_fingerprint' => ca_trusted_fingerprint) } if Gem::Version.create(LOGSTASH_VERSION) >= Gem::Version.create("8.3.0") it_behaves_like 'secured_elasticsearch' @@ -125,11 +125,11 @@ context 'setting host:port (and ssl)', secure_integration: true do - let(:client_options) { { :ca_file => ca_file, :user => user, :password => password } } + let(:client_options) { { :ssl_certificate_authorities => ca_file, :user => user, :password => password } } let(:config) do config = super().merge "hosts" => [ESHelper.get_host_port] - config.merge('user' => user, 'password' => password, 'ssl' => true, 'ca_file' => ca_file) + config.merge('user' => user, 'password' => password, 'ssl_enabled' => true, 'ssl_certificate_authorities' => ca_file) end it_behaves_like 'an elasticsearch index plugin' From fd9eef31c92e587483d99fa6f35171f5f19a0aba Mon Sep 17 00:00:00 2001 From: Edmo Vamerlatti Costa <11836452+edmocosta@users.noreply.github.com> Date: Mon, 6 Mar 2023 20:59:36 +0100 Subject: [PATCH 02/10] Update CHANGELOG.md --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9e0b8e9..3cbc0a77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,5 @@ ## 4.17.0 -- Added SSL settings for: [#1115](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1115) + - Added SSL settings for: [#185](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/185) - `ssl_enabled`: Enable/disable the SSL settings. If not provided, the value is inferred from the hosts scheme - `ssl_certificate`: OpenSSL-style X.509 certificate file to authenticate the client - `ssl_key`: OpenSSL-style RSA private key that corresponds to the `ssl_certificate` @@ -11,7 +11,7 @@ - `ssl_keystore_password`: The keystore password - `ssl_cipher_suites`: The list of cipher suites to use - `ssl_supported_protocols`: Supported protocols with versions -- Reviewed and deprecated SSL settings to comply with Logstash's naming convention + - Reviewed and deprecated SSL settings to comply with Logstash's naming convention - Deprecated `ssl` in favor of `ssl_enabled` - Deprecated `ca_file` in favor of `ssl_certificate_authorities` - Deprecated `ssl_certificate_verification` in favor of `ssl_verification_mode` From e30cbed69c18619546a3edf81236ef9dfe31ec3d Mon Sep 17 00:00:00 2001 From: edmocosta <11836452+edmocosta@users.noreply.github.com> Date: Tue, 7 Mar 2023 14:59:43 +0100 Subject: [PATCH 03/10] Fixed ssl_enabled inference and added more tests --- lib/logstash/inputs/elasticsearch.rb | 21 +++++++----- spec/inputs/elasticsearch_ssl_spec.rb | 49 +++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 8 deletions(-) diff --git a/lib/logstash/inputs/elasticsearch.rb b/lib/logstash/inputs/elasticsearch.rb index b18bc2b4..97a50827 100644 --- a/lib/logstash/inputs/elasticsearch.rb +++ b/lib/logstash/inputs/elasticsearch.rb @@ -259,8 +259,6 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base def initialize(params={}) super(params) - setup_ssl_params - if docinfo_target.nil? @docinfo_target = ecs_select[disabled: '@metadata', v1: '[@metadata][input][elasticsearch]'] end @@ -269,6 +267,9 @@ def initialize(params={}) def register require "rufus/scheduler" + fill_hosts_from_cloud_id + setup_ssl_params! + @options = { :index => @index, :scroll => @scroll, @@ -284,8 +285,6 @@ def register validate_authentication fill_user_password_from_cloud_auth - fill_hosts_from_cloud_id - transport_options = {:headers => {}} transport_options[:headers].merge!(setup_basic_auth(user, password)) @@ -467,12 +466,12 @@ def hosts_default?(hosts) end def effectively_ssl? - return @ssl_enabled unless @ssl_enabled.nil? + return true if @ssl_enabled hosts = Array(@hosts) return false if hosts.nil? || hosts.empty? - hosts.all? { |host| host && host.start_with?("https") } + hosts.all? { |host| host && host.to_s.start_with?("https") } end def validate_authentication @@ -552,13 +551,13 @@ def setup_client_ssl ssl_options end - def setup_ssl_params + def setup_ssl_params! @ssl_enabled = normalize_config(:ssl_enabled) do |normalize| normalize.with_deprecated_alias(:ssl) end # Infer the value if neither the deprecate `ssl` and `ssl_enabled` were set - @ssl_enabled = effectively_ssl? if @ssl_enabled.nil? + infer_ssl_enabled_from_hosts @ssl_certificate_authorities = normalize_config(:ssl_certificate_authorities) do |normalize| normalize.with_deprecated_mapping(:ca_file) do |ca_file| @@ -581,6 +580,12 @@ def setup_ssl_params params['ssl_verification_mode'] = @ssl_verification_mode unless @ssl_verification_mode.nil? end + def infer_ssl_enabled_from_hosts + return if original_params.include?('ssl') || original_params.include?('ssl_enabled') + + @ssl_enabled = params['ssl_enabled'] = effectively_ssl? + end + def setup_hosts @hosts = Array(@hosts).map { |host| host.to_s } # potential SafeURI#to_s @hosts.map do |h| diff --git a/spec/inputs/elasticsearch_ssl_spec.rb b/spec/inputs/elasticsearch_ssl_spec.rb index 96fd70b0..ea95c5ca 100644 --- a/spec/inputs/elasticsearch_ssl_spec.rb +++ b/spec/inputs/elasticsearch_ssl_spec.rb @@ -21,6 +21,32 @@ subject.close end + context "when ssl_enabled is" do + context "true and there is no https hosts" do + let(:hosts) { %w[http://es01 http://es01] } + + it "should not infer the ssl_enabled value" do + subject.register + expect(subject.instance_variable_get(:@ssl_enabled)).to eql(true) + expect(subject.params).to match hash_including("ssl_enabled" => true) + end + end + + context "false and cloud_id resolve host is https" do + let(:settings) {{ + "ssl_enabled" => false, + "hosts" => [], + "cloud_id" => "sample:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJGFjMzFlYmI5MDI0MTc3MzE1NzA0M2MzNGZkMjZmZDQ2OjkyNDMkYTRjMDYyMzBlNDhjOGZjZTdiZTg4YTA3NGEzYmIzZTA6OTI0NA==" + }} + + it "should not infer the ssl_enabled value" do + subject.register + expect(subject.instance_variable_get(:@ssl_enabled)).to eql(false) + expect(subject.params).to match hash_including("ssl_enabled" => false) + end + end + end + context "when neither ssl nor ssl_enabled is set" do let(:settings) { super().reject { |k| %w[ssl ssl_enabled].include?(k) } } @@ -43,6 +69,29 @@ expect(subject.params).to match hash_including("ssl_enabled" => true) end end + + context "and hosts have no scheme defined" do + let(:hosts) { %w[es01 es01] } + + it "should infer the ssl_enabled value to false" do + subject.register + expect(subject.instance_variable_get(:@ssl_enabled)).to eql(false) + expect(subject.params).to match hash_including("ssl_enabled" => false) + end + end + + context "and cloud_id resolved host is https" do + let(:settings) {{ + "hosts" => [], + "cloud_id" => "sample:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJGFjMzFlYmI5MDI0MTc3MzE1NzA0M2MzNGZkMjZmZDQ2OjkyNDMkYTRjMDYyMzBlNDhjOGZjZTdiZTg4YTA3NGEzYmIzZTA6OTI0NA==" + }} + + it "should infer the ssl_enabled value to false" do + subject.register + expect(subject.instance_variable_get(:@ssl_enabled)).to eql(true) + expect(subject.params).to match hash_including("ssl_enabled" => true) + end + end end context "when ssl_verification_mode" do From 04429ccadd1052212898c1a78ea94873e306a71c Mon Sep 17 00:00:00 2001 From: edmocosta <11836452+edmocosta@users.noreply.github.com> Date: Tue, 7 Mar 2023 17:34:39 +0100 Subject: [PATCH 04/10] [skip ci] Fixed typo on test --- spec/inputs/elasticsearch_ssl_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/inputs/elasticsearch_ssl_spec.rb b/spec/inputs/elasticsearch_ssl_spec.rb index ea95c5ca..6fa9551c 100644 --- a/spec/inputs/elasticsearch_ssl_spec.rb +++ b/spec/inputs/elasticsearch_ssl_spec.rb @@ -32,7 +32,7 @@ end end - context "false and cloud_id resolve host is https" do + context "false and cloud_id resolved host is https" do let(:settings) {{ "ssl_enabled" => false, "hosts" => [], From 8638f16636d3acf5dc2537447637f0991809089d Mon Sep 17 00:00:00 2001 From: Edmo Vamerlatti Costa <11836452+edmocosta@users.noreply.github.com> Date: Wed, 8 Mar 2023 16:02:04 +0100 Subject: [PATCH 05/10] Update index.asciidoc --- docs/index.asciidoc | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 58715616..0326e3df 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -472,7 +472,6 @@ More information on the importance of certificate verification: The list of cipher suites to use, listed by priorities. Supported cipher suites vary depending on the Java and protocol versions. - [id="plugins-{type}s-{plugin}-ssl_enabled"] ===== `ssl_enabled` From ee5c628661a5d9866d15073d4bfa58dd6a49ed8d Mon Sep 17 00:00:00 2001 From: edmocosta <11836452+edmocosta@users.noreply.github.com> Date: Thu, 9 Mar 2023 16:51:54 +0100 Subject: [PATCH 06/10] Apply PR suggestions --- docs/index.asciidoc | 28 +++++++------- lib/logstash/inputs/elasticsearch.rb | 56 ++++++++++++++-------------- 2 files changed, 42 insertions(+), 42 deletions(-) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 0326e3df..8cf69040 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -153,7 +153,7 @@ input plugins. * Value type is <> * There is no default value for this setting. -Authenticate using Elasticsearch API key. Note that this option also requires enabling the `ssl_enabled` option. +Authenticate using Elasticsearch API key. Note that this option also requires enabling the <> option. Format is `id:api_key` where `id` and `api_key` are as returned by the Elasticsearch @@ -162,7 +162,7 @@ API key API]. [id="plugins-{type}s-{plugin}-ca_file"] ===== `ca_file` -deprecated[8.8.0, Replaced by <>] +deprecated[4.17.0, Replaced by <>] * Value type is <> * There is no default value for this setting. @@ -422,7 +422,7 @@ instructions into the query. [id="plugins-{type}s-{plugin}-ssl"] ===== `ssl` -deprecated[8.8.0, Replaced by <>] +deprecated[4.17.0, Replaced by <>] * Value type is <> * Default value is `false` @@ -437,7 +437,7 @@ server (i.e. HTTPS will be used instead of plain HTTP). SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. -NOTE: This setting can be used only if `ssl_key` is set. +NOTE: This setting can be used only if <> is set. [id="plugins-{type}s-{plugin}-ssl_certificate_authorities"] ===== `ssl_certificate_authorities` @@ -445,13 +445,13 @@ NOTE: This setting can be used only if `ssl_key` is set. * Value type is a list of <> * There is no default value for this setting -The .cer or .pem files to validate the server's certificate. +The `.cer` or `.pem` files to validate the server's certificate. -NOTE: You cannot use this setting and `ssl_truststore_path` at the same time. +NOTE: You cannot use this setting and <> at the same time. [id="plugins-{type}s-{plugin}-ssl_certificate_verification"] ===== `ssl_certificate_verification` -deprecated[8.8.0, Replaced by <>] +deprecated[4.17.0, Replaced by <>] * Value type is <> * Default value is `true` @@ -487,9 +487,9 @@ If no explicit protocol is specified plain HTTP will be used. * Value type is <> * There is no default value for this setting. -OpenSSL-style RSA private key that corresponds to the `ssl_certificate`. +OpenSSL-style RSA private key that corresponds to the <>. -NOTE: This setting can be used only if `ssl_certificate` is set. +NOTE: This setting can be used only if <> is set. [id="plugins-{type}s-{plugin}-ssl_keystore_password"] ===== `ssl_keystore_password` @@ -506,9 +506,9 @@ Set the keystore password * There is no default value for this setting. The keystore used to present a certificate to the server. -It can be either .jks or .p12 +It can be either `.jks` or `.p12` -NOTE: You cannot use this setting and `ssl_certificate` at the same time. +NOTE: You cannot use this setting and <> at the same time. [id="plugins-{type}s-{plugin}-ssl_keystore_type"] ===== `ssl_keystore_type` @@ -528,7 +528,7 @@ The format of the keystore file. It must be either `jks` or `pkcs12`. List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. -For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the `LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, @@ -541,7 +541,7 @@ the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1. * Value type is <> * There is no default value for this setting. -Set the truststore password +Set the truststore password. [id="plugins-{type}s-{plugin}-ssl_truststore_path"] ===== `ssl_truststore_path` @@ -552,7 +552,7 @@ Set the truststore password The truststore to validate the server's certificate. It can be either .jks or .p12. -NOTE: You cannot use this setting and `ssl_certificate_authorities` at the same time. +NOTE: You cannot use this setting and <> at the same time. [id="plugins-{type}s-{plugin}-ssl_truststore_type"] ===== `ssl_truststore_type` diff --git a/lib/logstash/inputs/elasticsearch.rb b/lib/logstash/inputs/elasticsearch.rb index 97a50827..ee841570 100644 --- a/lib/logstash/inputs/elasticsearch.rb +++ b/lib/logstash/inputs/elasticsearch.rb @@ -190,12 +190,12 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base # SSL config :ssl, :validate => :boolean, :default => false, :deprecated => "Set 'ssl_enabled' instead." - # SSL - config :ssl_enabled, :validate => :boolean - - # SSL Certificate Authority file in PEM encoded format, must also include any chain certificates as necessary + # SSL Certificate Authority file in PEM encoded format, must also include any chain certificates as necessary config :ca_file, :validate => :path, :deprecated => "Set 'ssl_certificate_authorities' instead." + # OpenSSL-style X.509 certificate certificate to authenticate the client + config :ssl_certificate, :validate => :path + # SSL Certificate Authority files in PEM encoded format, must also include any chain certificates as necessary config :ssl_certificate_authorities, :validate => :path, :list => true @@ -204,27 +204,18 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base # https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf config :ssl_certificate_verification, :validate => :boolean, :default => true, :deprecated => "Set 'ssl_verification_mode' instead." - # Options to verify the server's certificate. - # "full": validates that the provided certificate has an issue date that’s within the not_before and not_after dates; - # chains to a trusted Certificate Authority (CA); has a hostname or IP address that matches the names within the certificate. - # "none": performs no certificate validation. Disabling this severely compromises security (https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf) - config :ssl_verification_mode, :validate => %w[full none], :default => 'full' + # The list of cipher suites to use, listed by priorities. + # Supported cipher suites vary depending on which version of Java is used. + config :ssl_cipher_suites, :validate => :string, :list => true - # OpenSSL-style X.509 certificate certificate to authenticate the client - config :ssl_certificate, :validate => :path + # SSL + config :ssl_enabled, :validate => :boolean # OpenSSL-style RSA private key to authenticate the client config :ssl_key, :validate => :path - # The JKS truststore to validate the server's certificate. - # Use either `:ssl_truststore_path` or `:ssl_certificate_authorities` - config :ssl_truststore_path, :validate => :path - - # The format of the truststore file. It must be either jks or pkcs12 - config :ssl_truststore_type, :validate => %w[pkcs12 jks] - - # Set the truststore password - config :ssl_truststore_password, :validate => :password + # Set the keystore password + config :ssl_keystore_password, :validate => :password # The keystore used to present a certificate to the server. # It can be either .jks or .p12 @@ -233,16 +224,25 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base # The format of the keystore file. It must be either jks or pkcs12 config :ssl_keystore_type, :validate => %w[pkcs12 jks] - # Set the keystore password - config :ssl_keystore_password, :validate => :password - - # The list of cipher suites to use, listed by priorities. - # Supported cipher suites vary depending on which version of Java is used. - config :ssl_cipher_suites, :validate => :string, :list => true - # Supported protocols with versions. config :ssl_supported_protocols, :validate => %w[TLSv1.1 TLSv1.2 TLSv1.3], :default => [], :list => true + # Set the truststore password + config :ssl_truststore_password, :validate => :password + + # The JKS truststore to validate the server's certificate. + # Use either `:ssl_truststore_path` or `:ssl_certificate_authorities` + config :ssl_truststore_path, :validate => :path + + # The format of the truststore file. It must be either jks or pkcs12 + config :ssl_truststore_type, :validate => %w[pkcs12 jks] + + # Options to verify the server's certificate. + # "full": validates that the provided certificate has an issue date that’s within the not_before and not_after dates; + # chains to a trusted Certificate Authority (CA); has a hostname or IP address that matches the names within the certificate. + # "none": performs no certificate validation. Disabling this severely compromises security (https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf) + config :ssl_verification_mode, :validate => %w[full none], :default => 'full' + # Schedule of when to periodically run statement, in Cron format # for example: "* * * * *" (execute query every minute, on the minute) # @@ -527,7 +527,7 @@ def setup_client_ssl if ssl_certificate && ssl_key ssl_options[:client_cert] = ssl_certificate ssl_options[:client_key] = ssl_key - elsif !!ssl_certificate ^ !!ssl_key + elsif !!ssl_certificate || !!ssl_key raise(LogStash::ConfigurationError, 'You must set both "ssl_certificate" and "ssl_key" for client authentication') end From 9ebb5fc25732dc98fac3844e7a9c1143a205828c Mon Sep 17 00:00:00 2001 From: edmocosta <11836452+edmocosta@users.noreply.github.com> Date: Thu, 9 Mar 2023 22:28:43 +0100 Subject: [PATCH 07/10] Apply PR suggestions --- lib/logstash/inputs/elasticsearch.rb | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/logstash/inputs/elasticsearch.rb b/lib/logstash/inputs/elasticsearch.rb index ee841570..3f6448ed 100644 --- a/lib/logstash/inputs/elasticsearch.rb +++ b/lib/logstash/inputs/elasticsearch.rb @@ -492,9 +492,12 @@ def validate_authentication def setup_client_ssl ssl_options = {} ssl_options[:ssl] = true if @ssl_enabled - ssl_options[:trust_strategy] = trust_strategy_for_ca_trusted_fingerprint - return ssl_options unless @ssl_enabled + unless @ssl_enabled + # Keep it backward compatible with the deprecated `ssl` option + ssl_options[:trust_strategy] = trust_strategy_for_ca_trusted_fingerprint if original_params.include?('ssl') + return ssl_options + end ssl_certificate_authorities, ssl_truststore_path, ssl_certificate, ssl_keystore_path = params.values_at('ssl_certificate_authorities', 'ssl_truststore_path', 'ssl_certificate', 'ssl_keystore_path') @@ -547,6 +550,7 @@ def setup_client_ssl protocols = params['ssl_supported_protocols'] ssl_options[:protocols] = protocols if protocols&.any? + ssl_options[:trust_strategy] = trust_strategy_for_ca_trusted_fingerprint ssl_options end @@ -575,7 +579,7 @@ def setup_ssl_params! end end - params['ssl_enabled'] = @ssl_enabled unless @ssl_enabled.nil? + params['ssl_enabled'] = @ssl_enabled params['ssl_certificate_authorities'] = @ssl_certificate_authorities unless @ssl_certificate_authorities.nil? params['ssl_verification_mode'] = @ssl_verification_mode unless @ssl_verification_mode.nil? end From 33159ada251ed1813693199f57292ec78363d280 Mon Sep 17 00:00:00 2001 From: edmocosta <11836452+edmocosta@users.noreply.github.com> Date: Fri, 10 Mar 2023 09:54:23 +0100 Subject: [PATCH 08/10] Fixing style guide for raise exceptions --- lib/logstash/inputs/elasticsearch.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/logstash/inputs/elasticsearch.rb b/lib/logstash/inputs/elasticsearch.rb index 3f6448ed..f6534ca5 100644 --- a/lib/logstash/inputs/elasticsearch.rb +++ b/lib/logstash/inputs/elasticsearch.rb @@ -502,15 +502,15 @@ def setup_client_ssl ssl_certificate_authorities, ssl_truststore_path, ssl_certificate, ssl_keystore_path = params.values_at('ssl_certificate_authorities', 'ssl_truststore_path', 'ssl_certificate', 'ssl_keystore_path') if ssl_certificate_authorities && ssl_truststore_path - raise(LogStash::ConfigurationError, 'Use either "ssl_certificate_authorities/ca_file" or "ssl_truststore_path" when configuring the CA certificate') + raise LogStash::ConfigurationError, 'Use either "ssl_certificate_authorities/ca_file" or "ssl_truststore_path" when configuring the CA certificate' end if ssl_certificate && ssl_keystore_path - raise(LogStash::ConfigurationError, 'Use either "ssl_certificate" or "ssl_keystore_path/keystore" when configuring client certificates') + raise LogStash::ConfigurationError, 'Use either "ssl_certificate" or "ssl_keystore_path/keystore" when configuring client certificates' end if ssl_certificate_authorities&.any? - raise(LogStash::ConfigurationError, 'Multiple values on "ssl_certificate_authorities" are not supported by this plugin') if ssl_certificate_authorities.size > 1 + raise LogStash::ConfigurationError, 'Multiple values on "ssl_certificate_authorities" are not supported by this plugin' if ssl_certificate_authorities.size > 1 ssl_options[:ca_file] = ssl_certificate_authorities.first end @@ -531,7 +531,7 @@ def setup_client_ssl ssl_options[:client_cert] = ssl_certificate ssl_options[:client_key] = ssl_key elsif !!ssl_certificate || !!ssl_key - raise(LogStash::ConfigurationError, 'You must set both "ssl_certificate" and "ssl_key" for client authentication') + raise LogStash::ConfigurationError, 'You must set both "ssl_certificate" and "ssl_key" for client authentication' end ssl_verification_mode = params["ssl_verification_mode"] From 4f291fa8c1a3d83337b038d84c35758262b30484 Mon Sep 17 00:00:00 2001 From: edmocosta <11836452+edmocosta@users.noreply.github.com> Date: Fri, 10 Mar 2023 11:23:49 +0100 Subject: [PATCH 09/10] Improving ssl_certificate/ssl_key validation and moving deprecated settings into a new section --- docs/index.asciidoc | 88 +++++++++++++++------------ lib/logstash/inputs/elasticsearch.rb | 7 ++- spec/inputs/elasticsearch_ssl_spec.rb | 4 +- 3 files changed, 56 insertions(+), 43 deletions(-) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 8cf69040..c44ab493 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -96,13 +96,12 @@ TIP: Set the `target` option to avoid potential schema conflicts. [id="plugins-{type}s-{plugin}-options"] ==== Elasticsearch Input configuration options -This plugin supports the following configuration options plus the <> described later. +This plugin supports the following configuration options plus the <> and the <> described later. [cols="<,<,<",options="header",] |======================================================================= |Setting |Input type|Required | <> |<>|No -| <> |a valid filesystem path|__Deprecated__ | <> |<>|No | <> |<>|No | <> |<>|No @@ -121,10 +120,8 @@ This plugin supports the following configuration options plus the <> |<>|No | <> |<>|No | <> |<>|No -| <> |<>|__Deprecated__ | <> |<>|No | <> |list of <>|No -| <> |<>|__Deprecated__ | <> |list of <>|No | <> |<>|No | <> |<>|No @@ -160,15 +157,6 @@ Elasticsearch {ref}/security-api-create-api-key.html[Create API key API]. -[id="plugins-{type}s-{plugin}-ca_file"] -===== `ca_file` -deprecated[4.17.0, Replaced by <>] - - * Value type is <> - * There is no default value for this setting. - -SSL Certificate Authority file in PEM encoded format, must also include any chain certificates as necessary. - [id="plugins-{type}s-{plugin}-ca_trusted_fingerprint"] ===== `ca_trusted_fingerprint` @@ -420,16 +408,6 @@ NOTE: The Elasticsearch manual indicates that there can be _negative_ performanc If the `slices` parameter is left unset, the plugin will _not_ inject slice instructions into the query. -[id="plugins-{type}s-{plugin}-ssl"] -===== `ssl` -deprecated[4.17.0, Replaced by <>] - - * Value type is <> - * Default value is `false` - -If enabled, SSL will be used when communicating with the Elasticsearch -server (i.e. HTTPS will be used instead of plain HTTP). - [id="plugins-{type}s-{plugin}-ssl_certificate"] ===== `ssl_certificate` * Value type is <> @@ -449,21 +427,6 @@ The `.cer` or `.pem` files to validate the server's certificate. NOTE: You cannot use this setting and <> at the same time. -[id="plugins-{type}s-{plugin}-ssl_certificate_verification"] -===== `ssl_certificate_verification` -deprecated[4.17.0, Replaced by <>] - - * Value type is <> - * Default value is `true` - -Option to validate the server's certificate. Disabling this severely compromises security. -When certificate validation is disabled, this plugin implicitly trusts the machine -resolved at the given address without validating its proof-of-identity. -In this scenario, the plugin can transmit credentials to or process data from an untrustworthy -man-in-the-middle or other compromised infrastructure. -More information on the importance of certificate verification: -**https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf**. - [id="plugins-{type}s-{plugin}-ssl_cipher_suites"] ===== `ssl_cipher_suites` * Value type is a list of <> @@ -612,6 +575,55 @@ option when authenticating to the Elasticsearch server. If set to an empty string authentication will be disabled. +[id="plugins-{type}s-{plugin}-deprecated-options"] +==== Elasticsearch Input deprecated configuration options + +This plugin supports the following deprecated configurations. + +WARNING: Deprecated options are subject to removal in future releases. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting|Input type|Replaced by +| <> |a valid filesystem path|<> +| <> |<>|<> +| <> |<>|<> +|======================================================================= + +[id="plugins-{type}s-{plugin}-ca_file"] +===== `ca_file` +deprecated[4.17.0, Replaced by <>] + +* Value type is <> +* There is no default value for this setting. + +SSL Certificate Authority file in PEM encoded format, must also include any chain certificates as necessary. + +[id="plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[4.17.0, Replaced by <>] + +* Value type is <> +* Default value is `false` + +If enabled, SSL will be used when communicating with the Elasticsearch +server (i.e. HTTPS will be used instead of plain HTTP). + + +[id="plugins-{type}s-{plugin}-ssl_certificate_verification"] +===== `ssl_certificate_verification` +deprecated[4.17.0, Replaced by <>] + +* Value type is <> +* Default value is `true` + +Option to validate the server's certificate. Disabling this severely compromises security. +When certificate validation is disabled, this plugin implicitly trusts the machine +resolved at the given address without validating its proof-of-identity. +In this scenario, the plugin can transmit credentials to or process data from an untrustworthy +man-in-the-middle or other compromised infrastructure. +More information on the importance of certificate verification: +**https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf**. [id="plugins-{type}s-{plugin}-common-options"] include::{include_path}/{type}.asciidoc[] diff --git a/lib/logstash/inputs/elasticsearch.rb b/lib/logstash/inputs/elasticsearch.rb index f6534ca5..1f3251ee 100644 --- a/lib/logstash/inputs/elasticsearch.rb +++ b/lib/logstash/inputs/elasticsearch.rb @@ -527,11 +527,12 @@ def setup_client_ssl end ssl_key = params["ssl_key"] - if ssl_certificate && ssl_key + if ssl_certificate + raise LogStash::ConfigurationError, 'Using an "ssl_certificate" requires an "ssl_key"' unless ssl_key ssl_options[:client_cert] = ssl_certificate ssl_options[:client_key] = ssl_key - elsif !!ssl_certificate || !!ssl_key - raise LogStash::ConfigurationError, 'You must set both "ssl_certificate" and "ssl_key" for client authentication' + elsif !ssl_key.nil? + raise LogStash::ConfigurationError, 'An "ssl_certificate" is required when using an "ssl_key"' end ssl_verification_mode = params["ssl_verification_mode"] diff --git a/spec/inputs/elasticsearch_ssl_spec.rb b/spec/inputs/elasticsearch_ssl_spec.rb index 6fa9551c..2189fd82 100644 --- a/spec/inputs/elasticsearch_ssl_spec.rb +++ b/spec/inputs/elasticsearch_ssl_spec.rb @@ -249,7 +249,7 @@ let(:settings) { super().reject { |k| "ssl_key".eql?(k) } } it "should raise a configuration error" do - expect { subject.register }.to raise_error(LogStash::ConfigurationError, /You must set both "ssl_certificate" and "ssl_key"/) + expect { subject.register }.to raise_error(LogStash::ConfigurationError, /Using an "ssl_certificate" requires an "ssl_key"/) end end @@ -257,7 +257,7 @@ let(:settings) { super().reject { |k| "ssl_certificate".eql?(k) } } it "should raise a configuration error" do - expect { subject.register }.to raise_error(LogStash::ConfigurationError, /You must set both "ssl_certificate" and "ssl_key"/) + expect { subject.register }.to raise_error(LogStash::ConfigurationError, /An "ssl_certificate" is required when using an "ssl_key"/) end end end From fecd2ba48c9b27123c2c825c340db7a81c19791c Mon Sep 17 00:00:00 2001 From: Edmo Vamerlatti Costa <11836452+edmocosta@users.noreply.github.com> Date: Fri, 10 Mar 2023 17:33:41 +0100 Subject: [PATCH 10/10] [skip ci] Fix typo in the CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3cbc0a77..fc7c7638 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ - `ssl_enabled`: Enable/disable the SSL settings. If not provided, the value is inferred from the hosts scheme - `ssl_certificate`: OpenSSL-style X.509 certificate file to authenticate the client - `ssl_key`: OpenSSL-style RSA private key that corresponds to the `ssl_certificate` - - `ssl_truststore_path`: he JKS truststore to validate the server's certificate + - `ssl_truststore_path`: The JKS truststore to validate the server's certificate - `ssl_truststore_type`: The format of the truststore file - `ssl_truststore_password`: The truststore password - `ssl_keystore_path`: The keystore used to present a certificate to the server