diff --git a/.ci/setup.sh b/.ci/setup.sh new file mode 100755 index 0000000..54a42db --- /dev/null +++ b/.ci/setup.sh @@ -0,0 +1,11 @@ +# user_agent requires /etc/protocols, which is provided by netbase. +# https://github.com/jruby/jruby/issues/3955 +if [ ! -f "/etc/protocols" ]; then + if [ $(command -v apt-get) ]; then + echo "installing netbase with apt-get" + sudo apt-get install -y netbase + else + echo "installing netbase with yum" + sudo yum install -y netbase + fi +fi \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e642c7..7774f4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 3.14.0 + - Added support for configurable retries with new `retry_on_failure` and `retry_on_status` options [#160](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/160) + ## 3.13.0 - Added support for this plugin identifying itself to Elasticsearch with an SSL/TLS client certificate using a new `keystore` option [#162](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/162) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index cafa705..8150034 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -142,6 +142,8 @@ This plugin supports the following configuration options plus the <> |<>|No | <> |<>|No | <> |<>|No +| <> |<>|No +| <> |<>|No | <> |<>|No | <> |<>|No | <> |a valid filesystem path|No @@ -330,11 +332,30 @@ the {ref}/query-dsl.html[Elasticsearch query documentation]. [id="plugins-{type}s-{plugin}-result_size"] ===== `result_size` - * Value type is <> - * Default value is `1` +* Value type is <> +* Default value is `1` How many results to return +[id="plugins-{type}s-{plugin}-retry_on_failure"] +===== `retry_on_failure` + +* Value type is <> +* Default value is `0` (retries disabled) + +How many times to retry an individual failed request. + +When enabled, retry requests that result in connection errors or an HTTP status code included in <> + +[id="plugins-{type}s-{plugin}-retry_on_status"] +===== `retry_on_status` + +* Value type is <> +* Default value is an empty list `[]` + +Which HTTP Status codes to consider for retries (in addition to connection errors) when using <>, + + [id="plugins-{type}s-{plugin}-sort"] ===== `sort` diff --git a/lib/logstash/filters/elasticsearch.rb b/lib/logstash/filters/elasticsearch.rb index e3b566c..38a9858 100644 --- a/lib/logstash/filters/elasticsearch.rb +++ b/lib/logstash/filters/elasticsearch.rb @@ -82,6 +82,12 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base # Tags the event on failure to look up geo information. This can be used in later analysis. config :tag_on_failure, :validate => :array, :default => ["_elasticsearch_lookup_failure"] + # How many times to retry on failure? + config :retry_on_failure, :validate => :number, :default => 0 + + # What status codes to retry on? + config :retry_on_status, :validate => :number, :list => true, :default => [500, 502, 503, 504] + # config :ca_trusted_fingerprint, :validate => :sha_256_hex include LogStash::PluginMixins::CATrustedFingerprintSupport @@ -215,6 +221,8 @@ def client_options :proxy => @proxy, :ssl => @ssl, :ca_file => @ca_file, + :retry_on_failure => @retry_on_failure, + :retry_on_status => @retry_on_status, :keystore => @keystore, :keystore_password => @keystore_password, :ssl_trust_strategy => trust_strategy_for_ca_trusted_fingerprint diff --git a/lib/logstash/filters/elasticsearch/client.rb b/lib/logstash/filters/elasticsearch/client.rb index 40354f1..4a46ac2 100644 --- a/lib/logstash/filters/elasticsearch/client.rb +++ b/lib/logstash/filters/elasticsearch/client.rb @@ -34,15 +34,23 @@ def initialize(logger, hosts, options = {}) # set ca_file even if ssl isn't on, since the host can be an https url ssl_options.update(ssl: true, ca_file: options[:ca_file]) if options[:ca_file] ssl_options.update(ssl: true, trust_strategy: options[:ssl_trust_strategy]) if options[:ssl_trust_strategy] - if keystore ssl_options[:keystore] = keystore logger.debug("Keystore for client certificate", :keystore => keystore) ssl_options[:keystore_password] = keystore_password.value if keystore_password end + client_options = { + hosts: hosts, + transport_class: ::Elasticsearch::Transport::Transport::HTTP::Manticore, + transport_options: transport_options, + ssl: ssl_options, + retry_on_failure: options[:retry_on_failure], + retry_on_status: options[:retry_on_status] + } + logger.info("New ElasticSearch filter client", :hosts => hosts) - @client = ::Elasticsearch::Client.new(hosts: hosts, transport_options: transport_options, transport_class: ::Elasticsearch::Transport::Transport::HTTP::Manticore, :ssl => ssl_options) + @client = ::Elasticsearch::Client.new(client_options) end def search(params) diff --git a/logstash-filter-elasticsearch.gemspec b/logstash-filter-elasticsearch.gemspec index 3cc4537..f5f6f60 100644 --- a/logstash-filter-elasticsearch.gemspec +++ b/logstash-filter-elasticsearch.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-filter-elasticsearch' - s.version = '3.13.0' + s.version = '3.14.0' s.licenses = ['Apache License (2.0)'] s.summary = "Copies fields from previous log events in Elasticsearch to current events " s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" @@ -29,4 +29,3 @@ Gem::Specification.new do |s| s.add_development_dependency 'logstash-devutils' end - diff --git a/spec/filters/elasticsearch_spec.rb b/spec/filters/elasticsearch_spec.rb index 484f808..74437e3 100644 --- a/spec/filters/elasticsearch_spec.rb +++ b/spec/filters/elasticsearch_spec.rb @@ -301,6 +301,15 @@ end end + context 'with client-level retries' do + let(:config) do + super().merge( + "retry_on_failure" => 3, + "retry_on_status" => [500] + ) + end + end + context "if query is on nested field" do let(:config) do { @@ -559,6 +568,28 @@ def wait_receive_request end end end + + describe "retry_on_failure" do + let(:config) { super().merge("retry_on_failure" => 3) } + + it 'propagates to the client' do + plugin.register + + client = plugin.send(:get_client).client + expect( extract_transport(client).options[:retry_on_failure] ).to eq(3) + end + end + + describe "retry_on_status" do + let(:config) { super().merge("retry_on_status" => [500, 502, 503, 504]) } + + it 'propagates to the client' do + plugin.register + + client = plugin.send(:get_client).client + expect( extract_transport(client).options[:retry_on_status] ).to eq([500, 502, 503, 504]) + end + end end describe "ca_trusted_fingerprint" do