From e88262c8338783007aa4d3706ec82fecbc1319c3 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Wed, 19 Sep 2018 17:55:31 -0400 Subject: [PATCH 01/19] Firat pass Basics work, code is rough, integration test passes --- lib/logstash/outputs/elasticsearch.rb | 19 ++++ lib/logstash/outputs/elasticsearch/common.rb | 24 +++-- .../outputs/elasticsearch/http_client.rb | 31 +++++++ .../outputs/elasticsearch/ilm_manager.rb | 64 +++++++++++++ .../outputs/elasticsearch/template_manager.rb | 6 ++ spec/es_spec_helper.rb | 39 ++++++++ spec/integration/outputs/ilm_spec.rb | 91 +++++++++++++++++++ .../api/actions/delete_ilm_policy.rb | 19 ++++ .../api/actions/get_ilm_policy.rb | 18 ++++ .../api/actions/put_ilm_policy.rb | 25 +++++ 10 files changed, 329 insertions(+), 7 deletions(-) create mode 100644 lib/logstash/outputs/elasticsearch/ilm_manager.rb create mode 100644 spec/integration/outputs/ilm_spec.rb create mode 100644 spec/support/elasticsearch/api/actions/delete_ilm_policy.rb create mode 100644 spec/support/elasticsearch/api/actions/get_ilm_policy.rb create mode 100644 spec/support/elasticsearch/api/actions/put_ilm_policy.rb diff --git a/lib/logstash/outputs/elasticsearch.rb b/lib/logstash/outputs/elasticsearch.rb index 69e5b1d66..5c7c6bd05 100644 --- a/lib/logstash/outputs/elasticsearch.rb +++ b/lib/logstash/outputs/elasticsearch.rb @@ -229,6 +229,25 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base # Custom Headers to send on each request to elasticsearch nodes config :custom_headers, :validate => :hash, :default => {} + + # ----- + # ILM configurations + # ----- + # Flag for enabling Index Lifecycle Management integration. + # TODO: Set this to false before commit + config :ilm_enabled, :validate => :boolean, :default => false + + # Write alias used for indexing data. If write alias doesn't exist, Logstash will create it and map it to the relevant index + # TODO: Remove write-alias + config :ilm_write_alias, :validate => :string, :default => 'logstash' + + # appends “000001” by default for new index creation, subsequent rollover indices will increment based on this pattern i.e. “000002” + config :ilm_pattern, :validate => :string, :default => '000001' + + # ILM policy to use, if undefined the default policy will be used. + config :ilm_policy, :validate => :string, :default => 'logstash-policy' + + def build_client params["metric"] = metric @client ||= ::LogStash::Outputs::ElasticSearch::HttpClientBuilder.build(@logger, @hosts, params) diff --git a/lib/logstash/outputs/elasticsearch/common.rb b/lib/logstash/outputs/elasticsearch/common.rb index 073438610..6ce186efd 100644 --- a/lib/logstash/outputs/elasticsearch/common.rb +++ b/lib/logstash/outputs/elasticsearch/common.rb @@ -1,8 +1,10 @@ require "logstash/outputs/elasticsearch/template_manager" +require "logstash/outputs/elasticsearch/ilm_manager" + module LogStash; module Outputs; class ElasticSearch; module Common - attr_reader :client, :hosts + attr_reader :client, :hosts, :ilm_manager # These codes apply to documents, not at the request level DOC_DLQ_CODES = [400, 404] @@ -30,6 +32,13 @@ def register @logger.info("New Elasticsearch output", :class => self.class.name, :hosts => @hosts.map(&:sanitized).map(&:to_s)) end + def setup_ilm + # ilm fields :ilm_enabled, :ilm_write_alias, :ilm_pattern, :ilm_policy + # As soon as the template is loaded, check for existence of write alias: + ILMManager.maybe_create_write_alias(self, @ilm_write_alias) + ILMManager.maybe_create_ilm_policy(self, @ilm_policy) + end + # Receive an array of events and immediately attempt to index them (no buffering) def multi_receive(events) until @template_installed.true? @@ -260,13 +269,13 @@ def get_event_type(event) type = if @document_type event.sprintf(@document_type) else - if client.maximum_seen_major_version < 6 - event.get("type") || DEFAULT_EVENT_TYPE_ES6 - elsif client.maximum_seen_major_version == 6 - DEFAULT_EVENT_TYPE_ES6 - else + # if client.maximum_seen_major_version < 6 + # event.get("type") || DEFAULT_EVENT_TYPE_ES6 + # elsif client.maximum_seen_major_version == 6 + # DEFAULT_EVENT_TYPE_ES6 + # else DEFAULT_EVENT_TYPE_ES7 - end + # end end if !(type.is_a?(String) || type.is_a?(Numeric)) @@ -281,6 +290,7 @@ def safe_bulk(actions) sleep_interval = @retry_initial_interval begin es_actions = actions.map {|action_type, params, event| [action_type, params, event.to_hash]} + @logger.error("There are #{actions.count} actions to do") response = @client.bulk(es_actions) response rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError => e diff --git a/lib/logstash/outputs/elasticsearch/http_client.rb b/lib/logstash/outputs/elasticsearch/http_client.rb index b46f4be0a..2be75e59f 100644 --- a/lib/logstash/outputs/elasticsearch/http_client.rb +++ b/lib/logstash/outputs/elasticsearch/http_client.rb @@ -348,6 +348,37 @@ def template_put(name, template) @pool.put(path, nil, LogStash::Json.dump(template)) end + def write_alias_exists?(name) + response = @pool.head("#{name}") + response.code >= 200 && response.code <= 299 + end + + def write_alias_put(name, alias_definition) + path = "#{name}" + logger.info("Creating write alias #{path}") + @pool.put(path, nil, LogStash::Json.dump(alias_definition)) + end + + def ilm_policy_exists?(name) + response = @pool.head("#{name}") + response.code >= 200 && response.code <= 299 + end + + + def ilm_policy_put(name, policy) + path = "_ilm/#{name}" + logger.info("Creating policy #{policy} at #{path}") + begin + response = @pool.put(path, nil, LogStash::Json.dump(policy)) + rescue => e + puts e.inspect + puts e.response_body + puts "request body: #{e.request_body}" + puts "ur: #{e.url}" + end + puts response + end + # Build a bulk item for an elasticsearch update action def update_action_builder(args, source) if args[:_script] diff --git a/lib/logstash/outputs/elasticsearch/ilm_manager.rb b/lib/logstash/outputs/elasticsearch/ilm_manager.rb new file mode 100644 index 000000000..ed68faa84 --- /dev/null +++ b/lib/logstash/outputs/elasticsearch/ilm_manager.rb @@ -0,0 +1,64 @@ +module LogStash; module Outputs; class ElasticSearch + class ILMManager + # config :ilm_enabled, :validate => :boolean, :default => false + # + # # Write alias used for indexing data. If write alias doesn't exist, Logstash will create it and map it to the relevant index + # config :ilm_write_alias, :validate => :string + # + # # appends “000001” by default for new index creation, subsequent rollover indices will increment based on this pattern i.e. “000002” + # config :ilm_pattern, :validate => :string, :default => '000001' + # + # # ILM policy to use, if undefined the default policy will be used. + # config :ilm_policy, :validate => :string, :default => 'logstash-policy' + + DEFAULT_POLICY="logstash-policy" + + def self.ilm_enabled?(plugin) + plugin.ilm_enabled + end + + def self.decorate_template(plugin, template) + # Include ilm settings in template: + template['settings'].update({ 'index.lifecycle.name' => plugin.ilm_policy, 'index.lifecycle.rollover_alias' => plugin.ilm_write_alias}) + end + + def self.maybe_create_write_alias(plugin, ilm_write_alias) + plugin.client.write_alias_put(write_alias_target(plugin), write_alias_payload(plugin)) unless plugin.client.write_alias_exists?(ilm_write_alias) + end + + def self.maybe_create_ilm_policy(plugin, ilm_policy) + plugin.client.ilm_policy_put(ilm_policy, default_policy_payload) if ilm_policy == DEFAULT_POLICY && !plugin.client.ilm_policy_exists?(ilm_policy) + end + + def self.write_alias_payload(plugin) + { + "aliases" => { + plugin.ilm_write_alias =>{ + "is_write_index" => true + } + } + } + end + + def self.write_alias_target(plugin) + "#{plugin.ilm_write_alias}-#{plugin.ilm_pattern}" + end + + def self.default_policy_payload + { + "policy" => { + "phases" => { + "hot" => { + "actions" => { + "rollover" => { + "max_size" => "25gb", + "max_age" => "30d" + } + } + } + } + } + } + end + end +end end end \ No newline at end of file diff --git a/lib/logstash/outputs/elasticsearch/template_manager.rb b/lib/logstash/outputs/elasticsearch/template_manager.rb index 47a74d328..a7503fb0b 100644 --- a/lib/logstash/outputs/elasticsearch/template_manager.rb +++ b/lib/logstash/outputs/elasticsearch/template_manager.rb @@ -5,9 +5,11 @@ def self.install_template(plugin) return unless plugin.manage_template plugin.logger.info("Using mapping template from", :path => plugin.template) template = get_template(plugin.template, plugin.maximum_seen_major_version) + decorate_template(plugin, template) plugin.logger.info("Attempting to install template", :manage_template => template) install(plugin.client, plugin.template_name, template, plugin.template_overwrite) rescue => e + puts e.backtrace plugin.logger.error("Failed to install template.", :message => e.message, :class => e.class.name, :backtrace => e.backtrace) end @@ -21,6 +23,10 @@ def self.install(client, template_name, template, template_overwrite) client.template_install(template_name, template, template_overwrite) end + def self.decorate_template(plugin, template) + ILMManager.decorate_template(plugin, template) #if @ilm_enabled + end + def self.default_template_path(es_major_version) template_version = es_major_version == 1 ? 2 : es_major_version default_template_name = "elasticsearch-template-es#{template_version}x.json" diff --git a/spec/es_spec_helper.rb b/spec/es_spec_helper.rb index 4960d254a..3d9550e72 100644 --- a/spec/es_spec_helper.rb +++ b/spec/es_spec_helper.rb @@ -1,6 +1,11 @@ require "logstash/devutils/rspec/spec_helper" require 'manticore' require 'elasticsearch' +require_relative "support/elasticsearch/api/actions/delete_ilm_policy" +require_relative "support/elasticsearch/api/actions/get_ilm_policy" +require_relative "support/elasticsearch/api/actions/put_ilm_policy" + +require 'json' module ESHelper def get_host_port @@ -61,6 +66,40 @@ def self.es_version_satisfies?(*requirement) es_release_version = Gem::Version.new(es_version).release Gem::Requirement.new(requirement).satisfied_by?(es_release_version) end + + def clean(client) + client.indices.delete_template(:name => "*") + # This can fail if there are no indexes, ignore failure. + client.indices.delete(:index => "*") rescue nil + clean_ilm(client) if supports_ilm?(client) + end + + def set_cluster_settings(client, cluster_settings) + client.cluster.put_settings(body: cluster_settings) + get_cluster_settings(client) + end + + def get_cluster_settings(client) + client.cluster.get_settings + end + + def put_policy(client, policy_name, policy) + client.put_ilm_policy({:name => policy_name, :body=> policy}) + end + + def clean_ilm(client) + client.get_ilm_policy.each_key {|key| client.delete_ilm_policy(name: key) } + end + + + def supports_ilm?(client) + begin + client.get_ilm_policy + true + rescue + false + end + end end RSpec.configure do |config| diff --git a/spec/integration/outputs/ilm_spec.rb b/spec/integration/outputs/ilm_spec.rb new file mode 100644 index 000000000..6b3d105bd --- /dev/null +++ b/spec/integration/outputs/ilm_spec.rb @@ -0,0 +1,91 @@ +require_relative "../../../spec/es_spec_helper" + + +# if ESHelper.supports_ilm?(get_client) +# if ESHelper.es_version_satisfies?(">= 5") + describe "Supports Index Lifecycle Management" do #, :integration => true do + let (:ilm_policy_name) {"three_and_done"} + subject! do + require "logstash/outputs/elasticsearch" + settings = { + "index" => "logstash", + "ilm_enabled" => true, + "ilm_policy" => ilm_policy_name, + "manage_template" => true, + "template_overwrite" => true, + "hosts" => "#{get_host_port()}" + } + next LogStash::Outputs::ElasticSearch.new(settings) + end + + before :each do + # Delete all templates first. + require "elasticsearch" + + # Clean ES of data before we start. + @es = get_client + clean(@es) + @old_cluster_settings = get_cluster_settings(@es) + set_cluster_settings(@es, {"persistent" => { + "indices.lifecycle.poll_interval" => "1s"} + }) + + + put_policy(@es,ilm_policy_name, {"policy" => { + "phases"=> { + "hot" => { + "actions" => { + "rollover" => { + "max_docs" => "3" + } + } + } + } + }}) + + + subject.register + + subject.multi_receive([ + LogStash::Event.new("message" => "sample message here"), + LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }), + LogStash::Event.new("somevalue" => 100), + ]) + + sleep(6) + + subject.multi_receive([ + LogStash::Event.new("country" => "us"), + LogStash::Event.new("country" => "at"), + LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] }) + ]) + + @es.indices.refresh + + # Wait or fail until everything's indexed. + Stud::try(20.times) do + r = @es.search + expect(r["hits"]["total"]).to eq(6) + end + end + + after :each do + set_cluster_settings(@es, @old_cluster_settings) + end + + # it 'should have a good template' do + # puts "the template is #{@es.indices.get_template(name: "logstash")}" + # expect(@es.indices.get_template(name: "logstash")).to eq("a hat") + # end + + it 'should rotate the indexes correctly' do + indexes_written = @es.search['hits']['hits'].each_with_object(Hash.new(0)) do |x, res| + index_written = x['_index'] + res[index_written] += 1 + end + puts indexes_written + expect(indexes_written['logstash-000001']).to eq(3) + expect(indexes_written['logstash-000002']).to eq(3) + end + + end diff --git a/spec/support/elasticsearch/api/actions/delete_ilm_policy.rb b/spec/support/elasticsearch/api/actions/delete_ilm_policy.rb new file mode 100644 index 000000000..05c529f46 --- /dev/null +++ b/spec/support/elasticsearch/api/actions/delete_ilm_policy.rb @@ -0,0 +1,19 @@ +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. + +module Elasticsearch + module API + module Actions + + # Update the password of the specified user + def delete_ilm_policy(arguments={}) + method = HTTP_DELETE + path = Utils.__pathify '_ilm/', + Utils.__escape(arguments[:name]) + params = {} + perform_request(method, path, params, nil).body + end + end + end +end \ No newline at end of file diff --git a/spec/support/elasticsearch/api/actions/get_ilm_policy.rb b/spec/support/elasticsearch/api/actions/get_ilm_policy.rb new file mode 100644 index 000000000..50d2fdff2 --- /dev/null +++ b/spec/support/elasticsearch/api/actions/get_ilm_policy.rb @@ -0,0 +1,18 @@ +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. + +module Elasticsearch + module API + module Actions + + # Retrieve the list of index lifecycle management policies + def get_ilm_policy(arguments={}) + method = HTTP_GET + path = Utils.__pathify '_ilm' + params = {} + perform_request(method, path, params, nil).body + end + end + end +end \ No newline at end of file diff --git a/spec/support/elasticsearch/api/actions/put_ilm_policy.rb b/spec/support/elasticsearch/api/actions/put_ilm_policy.rb new file mode 100644 index 000000000..111cbaa20 --- /dev/null +++ b/spec/support/elasticsearch/api/actions/put_ilm_policy.rb @@ -0,0 +1,25 @@ +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. + +module Elasticsearch + module API + module Actions + + # @option arguments [String] :name The name of the template (*Required*) + # @option arguments [Hash] :body The template definition (*Required*) + + def put_ilm_policy(arguments={}) + raise ArgumentError, "Required argument 'name' missing" unless arguments[:name] + raise ArgumentError, "Required argument 'body' missing" unless arguments[:body] + method = HTTP_PUT + path = Utils.__pathify '_ilm', Utils.__escape(arguments[:name]) + + params = Utils.__validate_and_extract_params arguments + + body = arguments[:body] + perform_request(method, path, params, body.to_json).body + end + end + end +end From c090462164d57d54a6962ab07b9bc52889cbfc96 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Thu, 20 Sep 2018 14:57:55 -0400 Subject: [PATCH 02/19] Reworked tests a bit --- lib/logstash/outputs/elasticsearch/common.rb | 10 +- .../outputs/elasticsearch/http_client.rb | 1 + .../outputs/elasticsearch/ilm_manager.rb | 1 + .../outputs/elasticsearch/template_manager.rb | 12 +- spec/es_spec_helper.rb | 7 +- spec/integration/outputs/ilm_spec.rb | 296 +++++++++++++++--- .../api/actions/get_ilm_policy.rb | 2 +- 7 files changed, 277 insertions(+), 52 deletions(-) diff --git a/lib/logstash/outputs/elasticsearch/common.rb b/lib/logstash/outputs/elasticsearch/common.rb index 6ce186efd..05ef4a932 100644 --- a/lib/logstash/outputs/elasticsearch/common.rb +++ b/lib/logstash/outputs/elasticsearch/common.rb @@ -25,16 +25,19 @@ def register setup_hosts # properly sets @hosts build_client + install_template_after_successful_connection check_action_validity @bulk_request_metrics = metric.namespace(:bulk_requests) @document_level_metrics = metric.namespace(:documents) - install_template_after_successful_connection + # setup_ilm if @ilm_enabled + # install_template_after_successful_connection @logger.info("New Elasticsearch output", :class => self.class.name, :hosts => @hosts.map(&:sanitized).map(&:to_s)) end def setup_ilm # ilm fields :ilm_enabled, :ilm_write_alias, :ilm_pattern, :ilm_policy # As soon as the template is loaded, check for existence of write alias: + ILMManager.maybe_create_write_alias(self, @ilm_write_alias) ILMManager.maybe_create_ilm_policy(self, @ilm_policy) end @@ -55,7 +58,10 @@ def install_template_after_successful_connection Stud.stoppable_sleep(sleep_interval) { @stopping.true? } sleep_interval = next_sleep_interval(sleep_interval) end - install_template if successful_connection? + if successful_connection? + install_template + setup_ilm if @ilm_enabled + end end end diff --git a/lib/logstash/outputs/elasticsearch/http_client.rb b/lib/logstash/outputs/elasticsearch/http_client.rb index 2be75e59f..29460e1ec 100644 --- a/lib/logstash/outputs/elasticsearch/http_client.rb +++ b/lib/logstash/outputs/elasticsearch/http_client.rb @@ -343,6 +343,7 @@ def template_exists?(name) end def template_put(name, template) + puts "******THE TEMPLATE NAME IS #{name}" path = "_template/#{name}" logger.info("Installing elasticsearch template to #{path}") @pool.put(path, nil, LogStash::Json.dump(template)) diff --git a/lib/logstash/outputs/elasticsearch/ilm_manager.rb b/lib/logstash/outputs/elasticsearch/ilm_manager.rb index ed68faa84..862e80316 100644 --- a/lib/logstash/outputs/elasticsearch/ilm_manager.rb +++ b/lib/logstash/outputs/elasticsearch/ilm_manager.rb @@ -19,6 +19,7 @@ def self.ilm_enabled?(plugin) def self.decorate_template(plugin, template) # Include ilm settings in template: + template['template'] = "#{plugin.ilm_write_alias}-*" template['settings'].update({ 'index.lifecycle.name' => plugin.ilm_policy, 'index.lifecycle.rollover_alias' => plugin.ilm_write_alias}) end diff --git a/lib/logstash/outputs/elasticsearch/template_manager.rb b/lib/logstash/outputs/elasticsearch/template_manager.rb index a7503fb0b..9735624d5 100644 --- a/lib/logstash/outputs/elasticsearch/template_manager.rb +++ b/lib/logstash/outputs/elasticsearch/template_manager.rb @@ -5,9 +5,10 @@ def self.install_template(plugin) return unless plugin.manage_template plugin.logger.info("Using mapping template from", :path => plugin.template) template = get_template(plugin.template, plugin.maximum_seen_major_version) - decorate_template(plugin, template) + decorate_template(plugin, template) if plugin.ilm_enabled plugin.logger.info("Attempting to install template", :manage_template => template) - install(plugin.client, plugin.template_name, template, plugin.template_overwrite) + # install(plugin.client, plugin.template_name, template, plugin.template_overwrite) + install(plugin.client, template_name(plugin), template, plugin.template_overwrite) rescue => e puts e.backtrace plugin.logger.error("Failed to install template.", :message => e.message, :class => e.class.name, :backtrace => e.backtrace) @@ -24,7 +25,12 @@ def self.install(client, template_name, template, template_overwrite) end def self.decorate_template(plugin, template) - ILMManager.decorate_template(plugin, template) #if @ilm_enabled + ILMManager.decorate_template(plugin, template) + end + + + def self.template_name(plugin) + plugin.ilm_enabled ? plugin.ilm_write_alias : plugin.template_name end def self.default_template_path(es_major_version) diff --git a/spec/es_spec_helper.rb b/spec/es_spec_helper.rb index 3d9550e72..58f9cbe30 100644 --- a/spec/es_spec_helper.rb +++ b/spec/es_spec_helper.rb @@ -83,15 +83,18 @@ def get_cluster_settings(client) client.cluster.get_settings end + def get_policy(client, policy_name) + client.get_ilm_policy(name: policy_name) + end + def put_policy(client, policy_name, policy) client.put_ilm_policy({:name => policy_name, :body=> policy}) end def clean_ilm(client) - client.get_ilm_policy.each_key {|key| client.delete_ilm_policy(name: key) } + client.get_ilm_policy.each_key {|key| client.delete_ilm_policy(name: key)} end - def supports_ilm?(client) begin client.get_ilm_policy diff --git a/spec/integration/outputs/ilm_spec.rb b/spec/integration/outputs/ilm_spec.rb index 6b3d105bd..2e00ba4e8 100644 --- a/spec/integration/outputs/ilm_spec.rb +++ b/spec/integration/outputs/ilm_spec.rb @@ -1,37 +1,202 @@ require_relative "../../../spec/es_spec_helper" -# if ESHelper.supports_ilm?(get_client) -# if ESHelper.es_version_satisfies?(">= 5") - describe "Supports Index Lifecycle Management" do #, :integration => true do - let (:ilm_policy_name) {"three_and_done"} - subject! do - require "logstash/outputs/elasticsearch" - settings = { - "index" => "logstash", - "ilm_enabled" => true, - "ilm_policy" => ilm_policy_name, + # describe "Supports Index Lifecycle Management" do #, :integration => true do + # let (:ilm_policy_name) {"three_and_done"} + # subject! do + # require "logstash/outputs/elasticsearch" + # settings = { + # "index" => "logstash", + # "ilm_enabled" => true, + # "ilm_policy" => ilm_policy_name, + # "manage_template" => true, + # "template_overwrite" => true, + # "hosts" => "#{get_host_port()}" + # } + # next LogStash::Outputs::ElasticSearch.new(settings) + # end + # + # before :each do + # # Delete all templates first. + # require "elasticsearch" + # + # # Clean ES of data before we start. + # @es = get_client + # clean(@es) + # @old_cluster_settings = get_cluster_settings(@es) + # set_cluster_settings(@es, {"persistent" => { + # "indices.lifecycle.poll_interval" => "1s"} + # }) + # + # + # put_policy(@es,ilm_policy_name, {"policy" => { + # "phases"=> { + # "hot" => { + # "actions" => { + # "rollover" => { + # "max_docs" => "3" + # } + # } + # } + # } + # }}) + # + # + # subject.register + # + # subject.multi_receive([ + # LogStash::Event.new("message" => "sample message here"), + # LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }), + # LogStash::Event.new("somevalue" => 100), + # ]) + # + # sleep(6) + # + # subject.multi_receive([ + # LogStash::Event.new("country" => "us"), + # LogStash::Event.new("country" => "at"), + # LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] }) + # ]) + # + # @es.indices.refresh + # + # # Wait or fail until everything's indexed. + # Stud::try(20.times) do + # r = @es.search + # expect(r["hits"]["total"]).to eq(6) + # end + # end + # + # after :each do + # set_cluster_settings(@es, @old_cluster_settings) + # clean(@es) + # end + # + # # it 'should have a good template' do + # # puts "the template is #{@es.indices.get_template(name: "logstash")}" + # # expect(@es.indices.get_template(name: "logstash")).to eq("a hat") + # # end + # + # it 'should install the default policy if it is not already installed' do + # + # end + # + # it 'should not install the default policy if it is not used' do + # + # end + # + # + # it 'should rotate the indexes correctly' do + # indexes_written = @es.search['hits']['hits'].each_with_object(Hash.new(0)) do |x, res| + # index_written = x['_index'] + # res[index_written] += 1 + # end + # puts indexes_written + # expect(indexes_written['logstash-000001']).to eq(3) + # expect(indexes_written['logstash-000002']).to eq(3) + # end + # + # end + + +describe "Simple Supports Index Lifecycle Management" do #, :integration => true do + require "logstash/outputs/elasticsearch" + let (:ilm_policy_name) {"three_and_done"} + let (:ilm_write_alias) { "the_write_alias" } + let (:settings) { + { + "index" => ilm_write_alias, + "ilm_enabled" => true, + "ilm_policy" => ilm_policy_name, + "ilm_write_alias" => ilm_write_alias, "manage_template" => true, "template_overwrite" => true, "hosts" => "#{get_host_port()}" - } - next LogStash::Outputs::ElasticSearch.new(settings) - end + } + } + let (:policy) { + {"policy" => { + "phases"=> { + "hot" => { + "actions" => { + "rollover" => { + "max_docs" => "3" + } + } + } + } + }} + } + subject do + # next + LogStash::Outputs::ElasticSearch.new(settings) + end - before :each do - # Delete all templates first. - require "elasticsearch" + before :each do + # Delete all templates first. + require "elasticsearch" - # Clean ES of data before we start. - @es = get_client - clean(@es) - @old_cluster_settings = get_cluster_settings(@es) - set_cluster_settings(@es, {"persistent" => { - "indices.lifecycle.poll_interval" => "1s"} - }) + # Clean ES of data before we start. + @es = get_client + clean(@es) + @old_cluster_settings = get_cluster_settings(@es) + set_cluster_settings(@es, {"persistent" => { + "indices.lifecycle.poll_interval" => "1s"} + }) + end + after :each do + set_cluster_settings(@es, @old_cluster_settings) + clean(@es) + end - put_policy(@es,ilm_policy_name, {"policy" => { + # it 'should have a good template' do + # puts "the template is #{@es.indices.get_template(name: "logstash")}" + # expect(@es.indices.get_template(name: "logstash")).to eq("a hat") + # end + + # it 'should install the default policy if it is not already installed' do + # subject.register + # + # subject.multi_receive([ + # LogStash::Event.new("message" => "sample message here"), + # LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }), + # LogStash::Event.new("somevalue" => 100), + # ]) + # + # sleep(6) + # + # subject.multi_receive([ + # LogStash::Event.new("country" => "us"), + # LogStash::Event.new("country" => "at"), + # LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] }) + # ]) + # + # @es.indices.refresh + # + # # Wait or fail until everything's indexed. + # Stud::try(20.times) do + # r = @es.search + # expect(r["hits"]["total"]).to eq(6) + # end + # end + + context 'when using the default policy' do + let (:ilm_policy_name) { LogStash::Outputs::ElasticSearch::ILMManager::DEFAULT_POLICY } + + it 'should install the default policy if it is used' do + expect{get_policy(@es, LogStash::Outputs::ElasticSearch::ILMManager::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) + subject.register + sleep(1) + expect{get_policy(@es, LogStash::Outputs::ElasticSearch::ILMManager::DEFAULT_POLICY)}.not_to raise_error + end + end + + + context 'when not using the default policy' do + let (:ilm_policy_name) {"new_one"} + let (:policy) {{ + "policy" => { "phases"=> { "hot" => { "actions" => { @@ -41,24 +206,46 @@ } } } - }}) + }}} + + before do + expect{get_policy(@es, LogStash::Outputs::ElasticSearch::ILMManager::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) + put_policy(@es,ilm_policy_name, policy) + end + + it 'should not install the default policy if it is not used' do + subject.register + sleep(1) + expect{get_policy(@es, LogStash::Outputs::ElasticSearch::ILMManager::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) + end + end + + context 'with ilm enabled' do + it 'should write the write alias' do + expect(@es.indices.exists_alias(index: ilm_write_alias)).to be_falsey + subject.register + sleep(3) + expect(@es.indices.exists_alias(index: ilm_write_alias)).to be_truthy + end + it 'should follow the policy' do + put_policy(@es,ilm_policy_name, policy) subject.register subject.multi_receive([ - LogStash::Event.new("message" => "sample message here"), - LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }), - LogStash::Event.new("somevalue" => 100), - ]) + LogStash::Event.new("message" => "sample message here"), + LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }), + LogStash::Event.new("somevalue" => 100), + ]) sleep(6) subject.multi_receive([ - LogStash::Event.new("country" => "us"), - LogStash::Event.new("country" => "at"), - LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] }) - ]) + LogStash::Event.new("country" => "us"), + LogStash::Event.new("country" => "at"), + LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] }) + ]) @es.indices.refresh @@ -67,25 +254,46 @@ r = @es.search expect(r["hits"]["total"]).to eq(6) end + indexes_written = @es.search['hits']['hits'].each_with_object(Hash.new(0)) do |x, res| + index_written = x['_index'] + res[index_written] += 1 + end + expect(indexes_written["#{ilm_write_alias}-000001"]).to eq(3) + expect(indexes_written["#{ilm_write_alias}-000002"]).to eq(3) end - after :each do - set_cluster_settings(@es, @old_cluster_settings) - end + it 'should index documents' do + # put_policy(@es,ilm_policy_name, policy) - # it 'should have a good template' do - # puts "the template is #{@es.indices.get_template(name: "logstash")}" - # expect(@es.indices.get_template(name: "logstash")).to eq("a hat") - # end + subject.register - it 'should rotate the indexes correctly' do + subject.multi_receive([ + LogStash::Event.new("message" => "sample message here"), + LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }), + LogStash::Event.new("somevalue" => 100), + ]) + + sleep(6) + + subject.multi_receive([ + LogStash::Event.new("country" => "us"), + LogStash::Event.new("country" => "at"), + LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] }) + ]) + + @es.indices.refresh + + # Wait or fail until everything's indexed. + Stud::try(20.times) do + r = @es.search + expect(r["hits"]["total"]).to eq(6) + end indexes_written = @es.search['hits']['hits'].each_with_object(Hash.new(0)) do |x, res| index_written = x['_index'] res[index_written] += 1 end - puts indexes_written - expect(indexes_written['logstash-000001']).to eq(3) - expect(indexes_written['logstash-000002']).to eq(3) + expect(indexes_written["#{ilm_write_alias}-000001"]).to eq(6) end end +end \ No newline at end of file diff --git a/spec/support/elasticsearch/api/actions/get_ilm_policy.rb b/spec/support/elasticsearch/api/actions/get_ilm_policy.rb index 50d2fdff2..eb993bb61 100644 --- a/spec/support/elasticsearch/api/actions/get_ilm_policy.rb +++ b/spec/support/elasticsearch/api/actions/get_ilm_policy.rb @@ -9,7 +9,7 @@ module Actions # Retrieve the list of index lifecycle management policies def get_ilm_policy(arguments={}) method = HTTP_GET - path = Utils.__pathify '_ilm' + path = Utils.__pathify '_ilm', Utils.__escape(arguments[:name]) params = {} perform_request(method, path, params, nil).body end From c08664940c0bc9295960bc2eead39d7666609d14 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Thu, 27 Sep 2018 09:42:17 -0400 Subject: [PATCH 03/19] WIP --- lib/logstash/outputs/elasticsearch.rb | 19 -- lib/logstash/outputs/elasticsearch/common.rb | 95 ++++-- .../outputs/elasticsearch/common_configs.rb | 18 ++ .../elasticsearch/default-ilm-policy.json | 14 + .../outputs/elasticsearch/http_client.rb | 44 +-- .../outputs/elasticsearch/ilm_manager.rb | 65 ---- .../outputs/elasticsearch/template_manager.rb | 13 +- spec/es_spec_helper.rb | 1 + spec/integration/outputs/ilm_spec.rb | 302 +++++++++--------- 9 files changed, 292 insertions(+), 279 deletions(-) create mode 100644 lib/logstash/outputs/elasticsearch/default-ilm-policy.json delete mode 100644 lib/logstash/outputs/elasticsearch/ilm_manager.rb diff --git a/lib/logstash/outputs/elasticsearch.rb b/lib/logstash/outputs/elasticsearch.rb index 5c7c6bd05..69e5b1d66 100644 --- a/lib/logstash/outputs/elasticsearch.rb +++ b/lib/logstash/outputs/elasticsearch.rb @@ -229,25 +229,6 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base # Custom Headers to send on each request to elasticsearch nodes config :custom_headers, :validate => :hash, :default => {} - - # ----- - # ILM configurations - # ----- - # Flag for enabling Index Lifecycle Management integration. - # TODO: Set this to false before commit - config :ilm_enabled, :validate => :boolean, :default => false - - # Write alias used for indexing data. If write alias doesn't exist, Logstash will create it and map it to the relevant index - # TODO: Remove write-alias - config :ilm_write_alias, :validate => :string, :default => 'logstash' - - # appends “000001” by default for new index creation, subsequent rollover indices will increment based on this pattern i.e. “000002” - config :ilm_pattern, :validate => :string, :default => '000001' - - # ILM policy to use, if undefined the default policy will be used. - config :ilm_policy, :validate => :string, :default => 'logstash-policy' - - def build_client params["metric"] = metric @client ||= ::LogStash::Outputs::ElasticSearch::HttpClientBuilder.build(@logger, @hosts, params) diff --git a/lib/logstash/outputs/elasticsearch/common.rb b/lib/logstash/outputs/elasticsearch/common.rb index 05ef4a932..7bbb9db89 100644 --- a/lib/logstash/outputs/elasticsearch/common.rb +++ b/lib/logstash/outputs/elasticsearch/common.rb @@ -1,5 +1,4 @@ require "logstash/outputs/elasticsearch/template_manager" -require "logstash/outputs/elasticsearch/ilm_manager" module LogStash; module Outputs; class ElasticSearch; @@ -10,6 +9,8 @@ module Common DOC_DLQ_CODES = [400, 404] DOC_SUCCESS_CODES = [200, 201] DOC_CONFLICT_CODE = 409 + DEFAULT_POLICY = "logstash-policy" + ILM_POLICY_PATH = "default-ilm-policy.json" # When you use external versioning, you are communicating that you want # to ignore conflicts. More obviously, since an external version is a @@ -19,6 +20,7 @@ module Common def register @template_installed = Concurrent::AtomicBoolean.new(false) + @ilm_verified = Concurrent::AtomicBoolean.new(false) @stopping = Concurrent::AtomicBoolean.new(false) # To support BWC, we check if DLQ exists in core (< 5.4). If it doesn't, we use nil to resort to previous behavior. @dlq_writer = dlq_enabled? ? execution_context.dlq_writer : nil @@ -29,19 +31,9 @@ def register check_action_validity @bulk_request_metrics = metric.namespace(:bulk_requests) @document_level_metrics = metric.namespace(:documents) - # setup_ilm if @ilm_enabled - # install_template_after_successful_connection @logger.info("New Elasticsearch output", :class => self.class.name, :hosts => @hosts.map(&:sanitized).map(&:to_s)) end - def setup_ilm - # ilm fields :ilm_enabled, :ilm_write_alias, :ilm_pattern, :ilm_policy - # As soon as the template is loaded, check for existence of write alias: - - ILMManager.maybe_create_write_alias(self, @ilm_write_alias) - ILMManager.maybe_create_ilm_policy(self, @ilm_policy) - end - # Receive an array of events and immediately attempt to index them (no buffering) def multi_receive(events) until @template_installed.true? @@ -59,8 +51,9 @@ def install_template_after_successful_connection sleep_interval = next_sleep_interval(sleep_interval) end if successful_connection? + verify_ilm install_template - setup_ilm if @ilm_enabled + setup_ilm end end end @@ -275,13 +268,13 @@ def get_event_type(event) type = if @document_type event.sprintf(@document_type) else - # if client.maximum_seen_major_version < 6 - # event.get("type") || DEFAULT_EVENT_TYPE_ES6 - # elsif client.maximum_seen_major_version == 6 - # DEFAULT_EVENT_TYPE_ES6 - # else + if client.maximum_seen_major_version < 6 + event.get("type") || DEFAULT_EVENT_TYPE_ES6 + elsif client.maximum_seen_major_version == 6 + DEFAULT_EVENT_TYPE_ES6 + else DEFAULT_EVENT_TYPE_ES7 - # end + end end if !(type.is_a?(String) || type.is_a?(Numeric)) @@ -296,7 +289,6 @@ def safe_bulk(actions) sleep_interval = @retry_initial_interval begin es_actions = actions.map {|action_type, params, event| [action_type, params, event.to_hash]} - @logger.error("There are #{actions.count} actions to do") response = @client.bulk(es_actions) response rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError => e @@ -362,11 +354,74 @@ def safe_bulk(actions) end end + def ilm_enabled? + @ilm_enabled + end + def dlq_enabled? # TODO there should be a better way to query if DLQ is enabled # See more in: https://github.com/elastic/logstash/issues/8064 respond_to?(:execution_context) && execution_context.respond_to?(:dlq_writer) && !execution_context.dlq_writer.inner_writer.is_a?(::LogStash::Util::DummyDeadLetterQueueWriter) end + + def setup_ilm + return unless ilm_enabled? + # ilm fields :ilm_enabled, :ilm_write_alias, :ilm_pattern, :ilm_policy + # As soon as the template is loaded, check for existence of write alias: + maybe_create_write_alias + maybe_create_ilm_policy + # maybe verify policy somewhere? + end + + def verify_ilm + return true unless ilm_enabled? + begin + + # For elasticsearch versions without ilm enablement, the _ilm endpoint will return a 400 bad request - the + # endpoint is interpreted as an index, and will return a bad request error as that is an illegal format for + # an index. + client.get_ilm_endpoint + rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e + if e.response_code == 404 + true + elsif e.response_code == 400 + false + raise LogStash::ConfigurationError, "THROW: ilm is enabled, but elasticsearch instance does not have ilm capability, #{e}" + else + raise e + end + end + + end + + def maybe_create_ilm_policy + if ilm_policy == DEFAULT_POLICY && !client.ilm_policy_exists?(ilm_policy) + client.ilm_policy_put(ilm_policy, policy_payload) + end + end + + def maybe_create_write_alias + client.write_alias_put(write_alias_target, write_alias_payload) unless client.write_alias_exists?(ilm_write_alias) + end + + def write_alias_target + "#{ilm_write_alias}-#{ilm_pattern}" + end + + def write_alias_payload + { + "aliases" => { + ilm_write_alias =>{ + "is_write_index" => true + } + } + } + end + + def policy_payload + policy_path = ::File.expand_path(ILM_POLICY_PATH, ::File.dirname(__FILE__)) + LogStash::Json.load(::IO.read(policy_path)) + end end -end; end; end +end end end diff --git a/lib/logstash/outputs/elasticsearch/common_configs.rb b/lib/logstash/outputs/elasticsearch/common_configs.rb index 2d0800863..6742724a4 100644 --- a/lib/logstash/outputs/elasticsearch/common_configs.rb +++ b/lib/logstash/outputs/elasticsearch/common_configs.rb @@ -136,6 +136,24 @@ def self.included(mod) # Set which ingest pipeline you wish to execute for an event. You can also use event dependent configuration # here like `pipeline => "%{INGEST_PIPELINE}"` mod.config :pipeline, :validate => :string, :default => nil + + + # ----- + # ILM configurations + # ----- + # Flag for enabling Index Lifecycle Management integration. + mod.config :ilm_enabled, :validate => :boolean, :default => false + + # Write alias used for indexing data. If write alias doesn't exist, Logstash will create it and map it to the relevant index + mod.config :ilm_write_alias, :validate => :string, :default => 'logstash' + + # appends “000001” by default for new index creation, subsequent rollover indices will increment based on this pattern i.e. “000002” + mod.config :ilm_pattern, :validate => :string, :default => '000001' + + # ILM policy to use, if undefined the default policy will be used. + mod.config :ilm_policy, :validate => :string, :default => 'logstash-policy' + + end end end end end diff --git a/lib/logstash/outputs/elasticsearch/default-ilm-policy.json b/lib/logstash/outputs/elasticsearch/default-ilm-policy.json new file mode 100644 index 000000000..4ba3b3f7f --- /dev/null +++ b/lib/logstash/outputs/elasticsearch/default-ilm-policy.json @@ -0,0 +1,14 @@ +{ + "policy" : { + "phases": { + "hot" : { + "actions" : { + "rollover" : { + "max_size" : "25gb", + "max_age":"30d" + } + } + } + } + } +} \ No newline at end of file diff --git a/lib/logstash/outputs/elasticsearch/http_client.rb b/lib/logstash/outputs/elasticsearch/http_client.rb index 29460e1ec..d0b48c6f8 100644 --- a/lib/logstash/outputs/elasticsearch/http_client.rb +++ b/lib/logstash/outputs/elasticsearch/http_client.rb @@ -337,49 +337,49 @@ def host_to_url(h) ::LogStash::Util::SafeURI.new(raw_url) end - def template_exists?(name) - response = @pool.head("/_template/#{name}") + def exists?(path, use_get=false) + response = use_get ? @pool.get(path) : @pool.head(path) response.code >= 200 && response.code <= 299 end + def template_exists?(name) + exists?("/_template/#{name}") + end + def template_put(name, template) - puts "******THE TEMPLATE NAME IS #{name}" path = "_template/#{name}" logger.info("Installing elasticsearch template to #{path}") @pool.put(path, nil, LogStash::Json.dump(template)) end + # ILM methods + + # check whether write alias already exists def write_alias_exists?(name) - response = @pool.head("#{name}") - response.code >= 200 && response.code <= 299 + exists?(name) end - def write_alias_put(name, alias_definition) - path = "#{name}" - logger.info("Creating write alias #{path}") - @pool.put(path, nil, LogStash::Json.dump(alias_definition)) + # Create a new write alias + def write_alias_put(alias_name, alias_definition) + logger.info("Creating write alias #{alias_name}") + @pool.put(alias_name, nil, LogStash::Json.dump(alias_definition)) end - def ilm_policy_exists?(name) - response = @pool.head("#{name}") - response.code >= 200 && response.code <= 299 + def get_ilm_endpoint + @pool.get(path) end + def ilm_policy_exists?(name) + exists?("/_ilm/#{name}", true) + end def ilm_policy_put(name, policy) path = "_ilm/#{name}" - logger.info("Creating policy #{policy} at #{path}") - begin - response = @pool.put(path, nil, LogStash::Json.dump(policy)) - rescue => e - puts e.inspect - puts e.response_body - puts "request body: #{e.request_body}" - puts "ur: #{e.url}" - end - puts response + logger.info("Installing ILM policy #{policy} to #{path}") + @pool.put(path, nil, LogStash::Json.dump(policy)) end + # Build a bulk item for an elasticsearch update action def update_action_builder(args, source) if args[:_script] diff --git a/lib/logstash/outputs/elasticsearch/ilm_manager.rb b/lib/logstash/outputs/elasticsearch/ilm_manager.rb deleted file mode 100644 index 862e80316..000000000 --- a/lib/logstash/outputs/elasticsearch/ilm_manager.rb +++ /dev/null @@ -1,65 +0,0 @@ -module LogStash; module Outputs; class ElasticSearch - class ILMManager - # config :ilm_enabled, :validate => :boolean, :default => false - # - # # Write alias used for indexing data. If write alias doesn't exist, Logstash will create it and map it to the relevant index - # config :ilm_write_alias, :validate => :string - # - # # appends “000001” by default for new index creation, subsequent rollover indices will increment based on this pattern i.e. “000002” - # config :ilm_pattern, :validate => :string, :default => '000001' - # - # # ILM policy to use, if undefined the default policy will be used. - # config :ilm_policy, :validate => :string, :default => 'logstash-policy' - - DEFAULT_POLICY="logstash-policy" - - def self.ilm_enabled?(plugin) - plugin.ilm_enabled - end - - def self.decorate_template(plugin, template) - # Include ilm settings in template: - template['template'] = "#{plugin.ilm_write_alias}-*" - template['settings'].update({ 'index.lifecycle.name' => plugin.ilm_policy, 'index.lifecycle.rollover_alias' => plugin.ilm_write_alias}) - end - - def self.maybe_create_write_alias(plugin, ilm_write_alias) - plugin.client.write_alias_put(write_alias_target(plugin), write_alias_payload(plugin)) unless plugin.client.write_alias_exists?(ilm_write_alias) - end - - def self.maybe_create_ilm_policy(plugin, ilm_policy) - plugin.client.ilm_policy_put(ilm_policy, default_policy_payload) if ilm_policy == DEFAULT_POLICY && !plugin.client.ilm_policy_exists?(ilm_policy) - end - - def self.write_alias_payload(plugin) - { - "aliases" => { - plugin.ilm_write_alias =>{ - "is_write_index" => true - } - } - } - end - - def self.write_alias_target(plugin) - "#{plugin.ilm_write_alias}-#{plugin.ilm_pattern}" - end - - def self.default_policy_payload - { - "policy" => { - "phases" => { - "hot" => { - "actions" => { - "rollover" => { - "max_size" => "25gb", - "max_age" => "30d" - } - } - } - } - } - } - end - end -end end end \ No newline at end of file diff --git a/lib/logstash/outputs/elasticsearch/template_manager.rb b/lib/logstash/outputs/elasticsearch/template_manager.rb index 9735624d5..b1790120a 100644 --- a/lib/logstash/outputs/elasticsearch/template_manager.rb +++ b/lib/logstash/outputs/elasticsearch/template_manager.rb @@ -5,9 +5,8 @@ def self.install_template(plugin) return unless plugin.manage_template plugin.logger.info("Using mapping template from", :path => plugin.template) template = get_template(plugin.template, plugin.maximum_seen_major_version) - decorate_template(plugin, template) if plugin.ilm_enabled + add_ilm_settings_to_template(plugin, template) if plugin.ilm_enabled? plugin.logger.info("Attempting to install template", :manage_template => template) - # install(plugin.client, plugin.template_name, template, plugin.template_overwrite) install(plugin.client, template_name(plugin), template, plugin.template_overwrite) rescue => e puts e.backtrace @@ -24,13 +23,15 @@ def self.install(client, template_name, template, template_overwrite) client.template_install(template_name, template, template_overwrite) end - def self.decorate_template(plugin, template) - ILMManager.decorate_template(plugin, template) + def self.add_ilm_settings_to_template(plugin, template) + # Include ilm settings in template: + template['template'] = "#{plugin.ilm_write_alias}-*" + template['settings'].update({ 'index.lifecycle.name' => plugin.ilm_policy, 'index.lifecycle.rollover_alias' => plugin.ilm_write_alias}) end - + # TODO: Verify this is ok def self.template_name(plugin) - plugin.ilm_enabled ? plugin.ilm_write_alias : plugin.template_name + plugin.ilm_enabled? ? plugin.ilm_write_alias : plugin.template_name end def self.default_template_path(es_major_version) diff --git a/spec/es_spec_helper.rb b/spec/es_spec_helper.rb index 58f9cbe30..370a6c0fb 100644 --- a/spec/es_spec_helper.rb +++ b/spec/es_spec_helper.rb @@ -68,6 +68,7 @@ def self.es_version_satisfies?(*requirement) end def clean(client) + client.indices.delete_template(:name => "*") # This can fail if there are no indexes, ignore failure. client.indices.delete(:index => "*") rescue nil diff --git a/spec/integration/outputs/ilm_spec.rb b/spec/integration/outputs/ilm_spec.rb index 2e00ba4e8..008102a5d 100644 --- a/spec/integration/outputs/ilm_spec.rb +++ b/spec/integration/outputs/ilm_spec.rb @@ -1,112 +1,16 @@ require_relative "../../../spec/es_spec_helper" - - # describe "Supports Index Lifecycle Management" do #, :integration => true do - # let (:ilm_policy_name) {"three_and_done"} - # subject! do - # require "logstash/outputs/elasticsearch" - # settings = { - # "index" => "logstash", - # "ilm_enabled" => true, - # "ilm_policy" => ilm_policy_name, - # "manage_template" => true, - # "template_overwrite" => true, - # "hosts" => "#{get_host_port()}" - # } - # next LogStash::Outputs::ElasticSearch.new(settings) - # end - # - # before :each do - # # Delete all templates first. - # require "elasticsearch" - # - # # Clean ES of data before we start. - # @es = get_client - # clean(@es) - # @old_cluster_settings = get_cluster_settings(@es) - # set_cluster_settings(@es, {"persistent" => { - # "indices.lifecycle.poll_interval" => "1s"} - # }) - # - # - # put_policy(@es,ilm_policy_name, {"policy" => { - # "phases"=> { - # "hot" => { - # "actions" => { - # "rollover" => { - # "max_docs" => "3" - # } - # } - # } - # } - # }}) - # - # - # subject.register - # - # subject.multi_receive([ - # LogStash::Event.new("message" => "sample message here"), - # LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }), - # LogStash::Event.new("somevalue" => 100), - # ]) - # - # sleep(6) - # - # subject.multi_receive([ - # LogStash::Event.new("country" => "us"), - # LogStash::Event.new("country" => "at"), - # LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] }) - # ]) - # - # @es.indices.refresh - # - # # Wait or fail until everything's indexed. - # Stud::try(20.times) do - # r = @es.search - # expect(r["hits"]["total"]).to eq(6) - # end - # end - # - # after :each do - # set_cluster_settings(@es, @old_cluster_settings) - # clean(@es) - # end - # - # # it 'should have a good template' do - # # puts "the template is #{@es.indices.get_template(name: "logstash")}" - # # expect(@es.indices.get_template(name: "logstash")).to eq("a hat") - # # end - # - # it 'should install the default policy if it is not already installed' do - # - # end - # - # it 'should not install the default policy if it is not used' do - # - # end - # - # - # it 'should rotate the indexes correctly' do - # indexes_written = @es.search['hits']['hits'].each_with_object(Hash.new(0)) do |x, res| - # index_written = x['_index'] - # res[index_written] += 1 - # end - # puts indexes_written - # expect(indexes_written['logstash-000001']).to eq(3) - # expect(indexes_written['logstash-000002']).to eq(3) - # end - # - # end - - -describe "Simple Supports Index Lifecycle Management" do #, :integration => true do +describe "ES supports Index Lifecycle Management" do #, :integration => true do require "logstash/outputs/elasticsearch" let (:ilm_policy_name) {"three_and_done"} let (:ilm_write_alias) { "the_write_alias" } + let (:index) { ilm_write_alias } + let (:ilm_enabled) { true } + let (:settings) { { - "index" => ilm_write_alias, - "ilm_enabled" => true, + "index" => index, + "ilm_enabled" => ilm_enabled, "ilm_policy" => ilm_policy_name, "ilm_write_alias" => ilm_write_alias, "manage_template" => true, @@ -127,10 +31,7 @@ } }} } - subject do - # next - LogStash::Outputs::ElasticSearch.new(settings) - end + subject { LogStash::Outputs::ElasticSearch.new(settings) } before :each do # Delete all templates first. @@ -140,9 +41,9 @@ @es = get_client clean(@es) @old_cluster_settings = get_cluster_settings(@es) - set_cluster_settings(@es, {"persistent" => { - "indices.lifecycle.poll_interval" => "1s"} - }) + # set_cluster_settings(@es, {"persistent" => { + # "indices.lifecycle.poll_interval" => "1s"} + # }) end after :each do @@ -155,40 +56,14 @@ # expect(@es.indices.get_template(name: "logstash")).to eq("a hat") # end - # it 'should install the default policy if it is not already installed' do - # subject.register - # - # subject.multi_receive([ - # LogStash::Event.new("message" => "sample message here"), - # LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }), - # LogStash::Event.new("somevalue" => 100), - # ]) - # - # sleep(6) - # - # subject.multi_receive([ - # LogStash::Event.new("country" => "us"), - # LogStash::Event.new("country" => "at"), - # LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] }) - # ]) - # - # @es.indices.refresh - # - # # Wait or fail until everything's indexed. - # Stud::try(20.times) do - # r = @es.search - # expect(r["hits"]["total"]).to eq(6) - # end - # end - context 'when using the default policy' do - let (:ilm_policy_name) { LogStash::Outputs::ElasticSearch::ILMManager::DEFAULT_POLICY } + let (:ilm_policy_name) { LogStash::Outputs::ElasticSearch::DEFAULT_POLICY } - it 'should install the default policy if it is used' do - expect{get_policy(@es, LogStash::Outputs::ElasticSearch::ILMManager::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) + it 'should install it if it is not present' do + expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) subject.register sleep(1) - expect{get_policy(@es, LogStash::Outputs::ElasticSearch::ILMManager::DEFAULT_POLICY)}.not_to raise_error + expect{get_policy(@es, ilm_policy_name)}.not_to raise_error end end @@ -209,26 +84,29 @@ }}} before do - expect{get_policy(@es, LogStash::Outputs::ElasticSearch::ILMManager::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) + expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) put_policy(@es,ilm_policy_name, policy) end it 'should not install the default policy if it is not used' do + subject.register + puts subject.policy_payload.to_json sleep(1) - expect{get_policy(@es, LogStash::Outputs::ElasticSearch::ILMManager::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) + expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) end end context 'with ilm enabled' do + let (:ilm_enabled) { true } it 'should write the write alias' do expect(@es.indices.exists_alias(index: ilm_write_alias)).to be_falsey subject.register - sleep(3) + sleep(1) expect(@es.indices.exists_alias(index: ilm_write_alias)).to be_truthy end - it 'should follow the policy' do + it 'should rollover when the policy max docs is reached' do put_policy(@es,ilm_policy_name, policy) subject.register @@ -247,24 +125,32 @@ LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] }) ]) + sleep(6) + + subject.multi_receive([ + LogStash::Event.new("country" => "uk"), + LogStash::Event.new("country" => "fr"), + LogStash::Event.new("geoip" => { "location" => [ 0.1, 1.0 ] }) + ]) + @es.indices.refresh # Wait or fail until everything's indexed. Stud::try(20.times) do r = @es.search - expect(r["hits"]["total"]).to eq(6) + expect(r["hits"]["total"]).to eq(9) end indexes_written = @es.search['hits']['hits'].each_with_object(Hash.new(0)) do |x, res| index_written = x['_index'] res[index_written] += 1 end + expect(indexes_written.count).to eq(3) expect(indexes_written["#{ilm_write_alias}-000001"]).to eq(3) expect(indexes_written["#{ilm_write_alias}-000002"]).to eq(3) + expect(indexes_written["#{ilm_write_alias}-000003"]).to eq(3) end - it 'should index documents' do - # put_policy(@es,ilm_policy_name, policy) - + it 'should ingest into a single index when max docs is not reached' do subject.register subject.multi_receive([ @@ -292,8 +178,130 @@ index_written = x['_index'] res[index_written] += 1 end + expect(indexes_written.count).to eq(1) expect(indexes_written["#{ilm_write_alias}-000001"]).to eq(6) end end + + context 'with ilm disabled' do + let (:ilm_enabled) { false } + + it 'should not write the write alias' do + expect(@es.indices.exists_alias(index: ilm_write_alias)).to be_falsey + subject.register + sleep(1) + expect(@es.indices.exists_alias(index: ilm_write_alias)).to be_falsey + end + + it 'should not install the default policy' do + subject.register + sleep(1) + expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) + end + + it 'should index documents normally' do + put_policy(@es,ilm_policy_name, policy) + + subject.register + + subject.multi_receive([ + LogStash::Event.new("message" => "sample message here"), + LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }), + LogStash::Event.new("somevalue" => 100), + ]) + + sleep(6) + + subject.multi_receive([ + LogStash::Event.new("country" => "us"), + LogStash::Event.new("country" => "at"), + LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] }) + ]) + + @es.indices.refresh + + # Wait or fail until everything's indexed. + Stud::try(20.times) do + r = @es.search + expect(r["hits"]["total"]).to eq(6) + end + indexes_written = @es.search['hits']['hits'].each_with_object(Hash.new(0)) do |x, res| + index_written = x['_index'] + res[index_written] += 1 + end + expect(indexes_written.count).to eq(1) + expect(indexes_written["#{index}"]).to eq(6) + end + end +end + +describe 'ES Does not support index lifecycle management' do + require "logstash/outputs/elasticsearch" + let (:ilm_policy_name) {"three_and_done"} + let (:ilm_write_alias) { "the_write_alias" } + let (:index) { "ilm_write_alias" } + let (:ilm_enabled) { false } + + let (:settings) { + { + "index" => index, + "ilm_enabled" => ilm_enabled, + "ilm_policy" => ilm_policy_name, + "ilm_write_alias" => ilm_write_alias, + "manage_template" => true, + "template_overwrite" => true, + "hosts" => "#{get_host_port()}" + } + } + + subject { LogStash::Outputs::ElasticSearch.new(settings) } + + + context 'when ilm if disabled' do + before :each do + require "elasticsearch" + + # Clean ES of data before we start. + @es = get_client + clean(@es) + end + + after :each do + # set_cluster_settings(@es, @old_cluster_settings) + clean(@es) + end + + it 'should index documents normally' do + subject.register + + subject.multi_receive([ + LogStash::Event.new("message" => "sample message here"), + LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }), + LogStash::Event.new("somevalue" => 100), + ]) + + sleep(6) + + subject.multi_receive([ + LogStash::Event.new("country" => "us"), + LogStash::Event.new("country" => "at"), + LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] }) + ]) + + @es.indices.refresh + + # Wait or fail until everything's indexed. + Stud::try(20.times) do + r = @es.search + expect(r["hits"]["total"]).to eq(6) + end + indexes_written = @es.search['hits']['hits'].each_with_object(Hash.new(0)) do |x, res| + index_written = x['_index'] + res[index_written] += 1 + end + expect(indexes_written.count).to eq(1) + expect(indexes_written["#{index}"]).to eq(6) + end + end end \ No newline at end of file From 7ccfc57798a43b0b922c49fac7dc3eeafffd34a2 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Fri, 28 Sep 2018 15:24:22 -0400 Subject: [PATCH 04/19] Make definition more strict and throw for inconsistencies modified: lib/logstash/outputs/elasticsearch/common_configs.rb --- lib/logstash/outputs/elasticsearch/common.rb | 25 +- .../outputs/elasticsearch/common_configs.rb | 4 +- .../elasticsearch-template-es6x.json | 3 +- spec/integration/outputs/ilm_spec.rb | 267 ++++++++++-------- 4 files changed, 174 insertions(+), 125 deletions(-) diff --git a/lib/logstash/outputs/elasticsearch/common.rb b/lib/logstash/outputs/elasticsearch/common.rb index 7bbb9db89..8928623ea 100644 --- a/lib/logstash/outputs/elasticsearch/common.rb +++ b/lib/logstash/outputs/elasticsearch/common.rb @@ -377,7 +377,7 @@ def setup_ilm def verify_ilm return true unless ilm_enabled? begin - + verify_ilm_config # For elasticsearch versions without ilm enablement, the _ilm endpoint will return a 400 bad request - the # endpoint is interpreted as an index, and will return a bad request error as that is an illegal format for # an index. @@ -387,16 +387,35 @@ def verify_ilm true elsif e.response_code == 400 false - raise LogStash::ConfigurationError, "THROW: ilm is enabled, but elasticsearch instance does not have ilm capability, #{e}" + raise LogStash::ConfigurationError, "Index Lifecycle Management feature is enabled, but Elasticsearch instance does not have ilm capability, #{e}" else raise e end end + end + + def verify_ilm_config + # Default index name is logstash-XXXX and the default write alias is logstash - should we just change overwrite the default index name here? + @index = @ilm_write_alias if @index == LogStash::Outputs::ElasticSearch::CommonConfigs::DEFAULT_INDEX_NAME + raise LogStash::ConfigurationError, "ILM configuration error: The index name #{@index} does not match the write alias #{@ilm_write_alias}" if @ilm_write_alias != @index + raise LogStash::ConfigurationError, "ILM configuration error: The template name #{@template_name} does not match the write alias #{@ilm_write_alias}" if @ilm_write_alias != @template_name + verify_ilm_policy unless ilm_policy_default? + end + + def ilm_policy_ok? + + end + + def ilm_policy_default? + ilm_policy == DEFAULT_POLICY + end + def verify_ilm_policy + raise LogStash::ConfigurationError, "The specified ILM policy does not exist" unless client.ilm_policy_exists?(ilm_policy) end def maybe_create_ilm_policy - if ilm_policy == DEFAULT_POLICY && !client.ilm_policy_exists?(ilm_policy) + if ilm_policy_default? && !client.ilm_policy_exists?(ilm_policy) client.ilm_policy_put(ilm_policy, policy_payload) end end diff --git a/lib/logstash/outputs/elasticsearch/common_configs.rb b/lib/logstash/outputs/elasticsearch/common_configs.rb index 6742724a4..4d5c9cbc7 100644 --- a/lib/logstash/outputs/elasticsearch/common_configs.rb +++ b/lib/logstash/outputs/elasticsearch/common_configs.rb @@ -2,6 +2,8 @@ module LogStash; module Outputs; class ElasticSearch module CommonConfigs + + DEFAULT_INDEX_NAME = "logstash-%{+YYYY.MM.dd}" def self.included(mod) # The index to write events to. This can be dynamic using the `%{foo}` syntax. # The default value will partition your indices by day so you can more easily @@ -10,7 +12,7 @@ def self.included(mod) # For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}. # LS uses Joda to format the index pattern from event timestamp. # Joda formats are defined http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[here]. - mod.config :index, :validate => :string, :default => "logstash-%{+YYYY.MM.dd}" + mod.config :index, :validate => :string, :default => DEFAULT_INDEX_NAME mod.config :document_type, :validate => :string, diff --git a/lib/logstash/outputs/elasticsearch/elasticsearch-template-es6x.json b/lib/logstash/outputs/elasticsearch/elasticsearch-template-es6x.json index 2227906eb..9cf70ceef 100644 --- a/lib/logstash/outputs/elasticsearch/elasticsearch-template-es6x.json +++ b/lib/logstash/outputs/elasticsearch/elasticsearch-template-es6x.json @@ -2,7 +2,8 @@ "template" : "logstash-*", "version" : 60001, "settings" : { - "index.refresh_interval" : "5s" + "index.refresh_interval" : "5s", + "number_of_shards": 1 }, "mappings" : { "_default_" : { diff --git a/spec/integration/outputs/ilm_spec.rb b/spec/integration/outputs/ilm_spec.rb index 008102a5d..4078a756a 100644 --- a/spec/integration/outputs/ilm_spec.rb +++ b/spec/integration/outputs/ilm_spec.rb @@ -2,7 +2,6 @@ describe "ES supports Index Lifecycle Management" do #, :integration => true do require "logstash/outputs/elasticsearch" - let (:ilm_policy_name) {"three_and_done"} let (:ilm_write_alias) { "the_write_alias" } let (:index) { ilm_write_alias } let (:ilm_enabled) { true } @@ -11,14 +10,16 @@ { "index" => index, "ilm_enabled" => ilm_enabled, - "ilm_policy" => ilm_policy_name, "ilm_write_alias" => ilm_write_alias, "manage_template" => true, + "template_name" => ilm_write_alias, "template_overwrite" => true, "hosts" => "#{get_host_port()}" } } - let (:policy) { + let (:policy) { small_max_doc_policy } + + let (:small_max_doc_policy) { {"policy" => { "phases"=> { "hot" => { @@ -31,6 +32,23 @@ } }} } + + let (:large_max_doc_policy) { + {"policy" => { + "phases"=> { + "hot" => { + "actions" => { + "rollover" => { + "max_docs" => "1000000" + } + } + } + } + }} + } + + + subject { LogStash::Outputs::ElasticSearch.new(settings) } before :each do @@ -41,9 +59,9 @@ @es = get_client clean(@es) @old_cluster_settings = get_cluster_settings(@es) - # set_cluster_settings(@es, {"persistent" => { - # "indices.lifecycle.poll_interval" => "1s"} - # }) + set_cluster_settings(@es, {"persistent" => { + "indices.lifecycle.poll_interval" => "1s"} + }) end after :each do @@ -51,25 +69,20 @@ clean(@es) end - # it 'should have a good template' do - # puts "the template is #{@es.indices.get_template(name: "logstash")}" - # expect(@es.indices.get_template(name: "logstash")).to eq("a hat") - # end - - context 'when using the default policy' do - let (:ilm_policy_name) { LogStash::Outputs::ElasticSearch::DEFAULT_POLICY } + context 'when using the default policy' do\ it 'should install it if it is not present' do expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) subject.register sleep(1) - expect{get_policy(@es, ilm_policy_name)}.not_to raise_error + expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.not_to raise_error end end context 'when not using the default policy' do let (:ilm_policy_name) {"new_one"} + let (:settings) { super.merge("ilm_policy" => ilm_policy_name)} let (:policy) {{ "policy" => { "phases"=> { @@ -89,9 +102,7 @@ end it 'should not install the default policy if it is not used' do - subject.register - puts subject.policy_payload.to_json sleep(1) expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) end @@ -99,6 +110,7 @@ context 'with ilm enabled' do let (:ilm_enabled) { true } + it 'should write the write alias' do expect(@es.indices.exists_alias(index: ilm_write_alias)).to be_falsey subject.register @@ -106,82 +118,93 @@ expect(@es.indices.exists_alias(index: ilm_write_alias)).to be_truthy end - it 'should rollover when the policy max docs is reached' do - put_policy(@es,ilm_policy_name, policy) - - subject.register - - subject.multi_receive([ - LogStash::Event.new("message" => "sample message here"), - LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }), - LogStash::Event.new("somevalue" => 100), - ]) - - sleep(6) - - subject.multi_receive([ - LogStash::Event.new("country" => "us"), - LogStash::Event.new("country" => "at"), - LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] }) - ]) - - sleep(6) - - subject.multi_receive([ - LogStash::Event.new("country" => "uk"), - LogStash::Event.new("country" => "fr"), - LogStash::Event.new("geoip" => { "location" => [ 0.1, 1.0 ] }) - ]) - - @es.indices.refresh - - # Wait or fail until everything's indexed. - Stud::try(20.times) do - r = @es.search - expect(r["hits"]["total"]).to eq(9) + context 'with a policy with a maximum number of documents' do + let (:policy) { small_max_doc_policy } + let (:ilm_policy_name) { "few_docs"} + let (:settings) { super.merge("ilm_policy" => ilm_policy_name)} + + it 'should rollover when the policy max docs is reached' do + put_policy(@es,ilm_policy_name, policy) + subject.register + + subject.multi_receive([ + LogStash::Event.new("message" => "sample message here"), + LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }), + LogStash::Event.new("somevalue" => 100), + ]) + + sleep(6) + + subject.multi_receive([ + LogStash::Event.new("country" => "us"), + LogStash::Event.new("country" => "at"), + LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] }) + ]) + + sleep(6) + + subject.multi_receive([ + LogStash::Event.new("country" => "uk"), + LogStash::Event.new("country" => "fr"), + LogStash::Event.new("geoip" => { "location" => [ 0.1, 1.0 ] }) + ]) + + @es.indices.refresh + + # Wait or fail until everything's indexed. + Stud::try(20.times) do + r = @es.search + expect(r["hits"]["total"]).to eq(9) + end + indexes_written = @es.search['hits']['hits'].each_with_object(Hash.new(0)) do |x, res| + index_written = x['_index'] + res[index_written] += 1 + end + expect(indexes_written.count).to eq(3) + expect(indexes_written["#{ilm_write_alias}-000001"]).to eq(3) + expect(indexes_written["#{ilm_write_alias}-000002"]).to eq(3) + expect(indexes_written["#{ilm_write_alias}-000003"]).to eq(3) end - indexes_written = @es.search['hits']['hits'].each_with_object(Hash.new(0)) do |x, res| - index_written = x['_index'] - res[index_written] += 1 - end - expect(indexes_written.count).to eq(3) - expect(indexes_written["#{ilm_write_alias}-000001"]).to eq(3) - expect(indexes_written["#{ilm_write_alias}-000002"]).to eq(3) - expect(indexes_written["#{ilm_write_alias}-000003"]).to eq(3) end - it 'should ingest into a single index when max docs is not reached' do - subject.register - - subject.multi_receive([ - LogStash::Event.new("message" => "sample message here"), - LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }), - LogStash::Event.new("somevalue" => 100), - ]) - - sleep(6) - - subject.multi_receive([ - LogStash::Event.new("country" => "us"), - LogStash::Event.new("country" => "at"), - LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] }) - ]) - - @es.indices.refresh - - # Wait or fail until everything's indexed. - Stud::try(20.times) do - r = @es.search - expect(r["hits"]["total"]).to eq(6) - end - indexes_written = @es.search['hits']['hits'].each_with_object(Hash.new(0)) do |x, res| - index_written = x['_index'] - res[index_written] += 1 + context 'with a policy where the maximum number of documents is not reached' do + let (:policy) { large_max_doc_policy } + let (:ilm_policy_name) { "many_docs"} + let (:settings) { super.merge("ilm_policy" => ilm_policy_name)} + + it 'should ingest into a single index when max docs is not reached' do + put_policy(@es,ilm_policy_name, policy) + subject.register + + subject.multi_receive([ + LogStash::Event.new("message" => "sample message here"), + LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }), + LogStash::Event.new("somevalue" => 100), + ]) + + sleep(6) + + subject.multi_receive([ + LogStash::Event.new("country" => "us"), + LogStash::Event.new("country" => "at"), + LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] }) + ]) + + @es.indices.refresh + + # Wait or fail until everything's indexed. + Stud::try(20.times) do + r = @es.search + expect(r["hits"]["total"]).to eq(6) + end + indexes_written = @es.search['hits']['hits'].each_with_object(Hash.new(0)) do |x, res| + index_written = x['_index'] + res[index_written] += 1 + end + expect(indexes_written.count).to eq(1) + expect(indexes_written["#{ilm_write_alias}-000001"]).to eq(6) end - expect(indexes_written.count).to eq(1) - expect(indexes_written["#{ilm_write_alias}-000001"]).to eq(6) end - end context 'with ilm disabled' do @@ -200,45 +223,49 @@ expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) end - it 'should index documents normally' do - put_policy(@es,ilm_policy_name, policy) - - subject.register - - subject.multi_receive([ - LogStash::Event.new("message" => "sample message here"), - LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }), - LogStash::Event.new("somevalue" => 100), - ]) - - sleep(6) - - subject.multi_receive([ - LogStash::Event.new("country" => "us"), - LogStash::Event.new("country" => "at"), - LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] }) - ]) - - @es.indices.refresh - - # Wait or fail until everything's indexed. - Stud::try(20.times) do - r = @es.search - expect(r["hits"]["total"]).to eq(6) + context 'with an existing policy that will roll over' do + let (:policy) { small_max_doc_policy } + let (:ilm_policy_name) { "3_docs"} + let (:settings) { super.merge("ilm_policy" => ilm_policy_name)} + + it 'should index documents normally' do + + subject.register + subject.multi_receive([ + LogStash::Event.new("message" => "sample message here"), + LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }), + LogStash::Event.new("somevalue" => 100), + ]) + + sleep(6) + + subject.multi_receive([ + LogStash::Event.new("country" => "us"), + LogStash::Event.new("country" => "at"), + LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] }) + ]) + + @es.indices.refresh + + # Wait or fail until everything's indexed. + Stud::try(20.times) do + r = @es.search + expect(r["hits"]["total"]).to eq(6) + end + indexes_written = @es.search['hits']['hits'].each_with_object(Hash.new(0)) do |x, res| + index_written = x['_index'] + res[index_written] += 1 + end + expect(indexes_written.count).to eq(1) + expect(indexes_written["#{index}"]).to eq(6) end - indexes_written = @es.search['hits']['hits'].each_with_object(Hash.new(0)) do |x, res| - index_written = x['_index'] - res[index_written] += 1 - end - expect(indexes_written.count).to eq(1) - expect(indexes_written["#{index}"]).to eq(6) end end end describe 'ES Does not support index lifecycle management' do require "logstash/outputs/elasticsearch" - let (:ilm_policy_name) {"three_and_done"} + # let (:ilm_policy_name) {"three_and_done"} let (:ilm_write_alias) { "the_write_alias" } let (:index) { "ilm_write_alias" } let (:ilm_enabled) { false } @@ -247,7 +274,7 @@ { "index" => index, "ilm_enabled" => ilm_enabled, - "ilm_policy" => ilm_policy_name, + # "ilm_policy" => ilm_policy_name, "ilm_write_alias" => ilm_write_alias, "manage_template" => true, "template_overwrite" => true, From f945a13895c9e78f280ef6e53fa1633175db1608 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Wed, 3 Oct 2018 14:05:02 -0400 Subject: [PATCH 05/19] Add further tests --- lib/logstash/outputs/elasticsearch/common.rb | 4 +- .../outputs/elasticsearch/http_client.rb | 4 +- .../outputs/elasticsearch/template_manager.rb | 1 - spec/es_spec_helper.rb | 1 - spec/integration/outputs/ilm_spec.rb | 550 ++++++++++-------- 5 files changed, 309 insertions(+), 251 deletions(-) diff --git a/lib/logstash/outputs/elasticsearch/common.rb b/lib/logstash/outputs/elasticsearch/common.rb index 8928623ea..c72e45e7d 100644 --- a/lib/logstash/outputs/elasticsearch/common.rb +++ b/lib/logstash/outputs/elasticsearch/common.rb @@ -1,4 +1,5 @@ require "logstash/outputs/elasticsearch/template_manager" +require 'logstash/environment' module LogStash; module Outputs; class ElasticSearch; @@ -367,11 +368,9 @@ def dlq_enabled? def setup_ilm return unless ilm_enabled? - # ilm fields :ilm_enabled, :ilm_write_alias, :ilm_pattern, :ilm_policy # As soon as the template is loaded, check for existence of write alias: maybe_create_write_alias maybe_create_ilm_policy - # maybe verify policy somewhere? end def verify_ilm @@ -382,6 +381,7 @@ def verify_ilm # endpoint is interpreted as an index, and will return a bad request error as that is an illegal format for # an index. client.get_ilm_endpoint + true rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e if e.response_code == 404 true diff --git a/lib/logstash/outputs/elasticsearch/http_client.rb b/lib/logstash/outputs/elasticsearch/http_client.rb index d0b48c6f8..d911ac3d6 100644 --- a/lib/logstash/outputs/elasticsearch/http_client.rb +++ b/lib/logstash/outputs/elasticsearch/http_client.rb @@ -365,8 +365,8 @@ def write_alias_put(alias_name, alias_definition) @pool.put(alias_name, nil, LogStash::Json.dump(alias_definition)) end - def get_ilm_endpoint - @pool.get(path) + def get_ilm_endpoint + @pool.get("/_ilm") end def ilm_policy_exists?(name) diff --git a/lib/logstash/outputs/elasticsearch/template_manager.rb b/lib/logstash/outputs/elasticsearch/template_manager.rb index b1790120a..d76b8465e 100644 --- a/lib/logstash/outputs/elasticsearch/template_manager.rb +++ b/lib/logstash/outputs/elasticsearch/template_manager.rb @@ -9,7 +9,6 @@ def self.install_template(plugin) plugin.logger.info("Attempting to install template", :manage_template => template) install(plugin.client, template_name(plugin), template, plugin.template_overwrite) rescue => e - puts e.backtrace plugin.logger.error("Failed to install template.", :message => e.message, :class => e.class.name, :backtrace => e.backtrace) end diff --git a/spec/es_spec_helper.rb b/spec/es_spec_helper.rb index 370a6c0fb..58f9cbe30 100644 --- a/spec/es_spec_helper.rb +++ b/spec/es_spec_helper.rb @@ -68,7 +68,6 @@ def self.es_version_satisfies?(*requirement) end def clean(client) - client.indices.delete_template(:name => "*") # This can fail if there are no indexes, ignore failure. client.indices.delete(:index => "*") rescue nil diff --git a/spec/integration/outputs/ilm_spec.rb b/spec/integration/outputs/ilm_spec.rb index 4078a756a..d7ab1eee8 100644 --- a/spec/integration/outputs/ilm_spec.rb +++ b/spec/integration/outputs/ilm_spec.rb @@ -1,334 +1,394 @@ require_relative "../../../spec/es_spec_helper" -describe "ES supports Index Lifecycle Management" do #, :integration => true do +shared_examples_for 'an Elasticsearch instance that does not support index lifecycle management' do require "logstash/outputs/elasticsearch" - let (:ilm_write_alias) { "the_write_alias" } - let (:index) { ilm_write_alias } - let (:ilm_enabled) { true } + let (:ilm_enabled) { false } let (:settings) { { - "index" => index, "ilm_enabled" => ilm_enabled, - "ilm_write_alias" => ilm_write_alias, - "manage_template" => true, - "template_name" => ilm_write_alias, - "template_overwrite" => true, "hosts" => "#{get_host_port()}" } } - let (:policy) { small_max_doc_policy } - - let (:small_max_doc_policy) { - {"policy" => { - "phases"=> { - "hot" => { - "actions" => { - "rollover" => { - "max_docs" => "3" - } - } - } - } - }} - } - - let (:large_max_doc_policy) { - {"policy" => { - "phases"=> { - "hot" => { - "actions" => { - "rollover" => { - "max_docs" => "1000000" - } - } - } - } - }} - } - - - - subject { LogStash::Outputs::ElasticSearch.new(settings) } before :each do - # Delete all templates first. require "elasticsearch" # Clean ES of data before we start. @es = get_client clean(@es) - @old_cluster_settings = get_cluster_settings(@es) - set_cluster_settings(@es, {"persistent" => { - "indices.lifecycle.poll_interval" => "1s"} - }) end after :each do - set_cluster_settings(@es, @old_cluster_settings) clean(@es) end + subject { LogStash::Outputs::ElasticSearch.new(settings) } - context 'when using the default policy' do\ - it 'should install it if it is not present' do - expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) - subject.register - sleep(1) - expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.not_to raise_error + context 'when ilm is enabled' do + let (:ilm_enabled) { true } + + it 'should raise a configuration error' do + expect do + begin + subject.register + ensure + subject.stop_template_installer + end + end.to raise_error(LogStash::ConfigurationError) end end + context 'when ilm is disabled' do + it 'should index documents normally' do + subject.register - context 'when not using the default policy' do - let (:ilm_policy_name) {"new_one"} - let (:settings) { super.merge("ilm_policy" => ilm_policy_name)} - let (:policy) {{ - "policy" => { - "phases"=> { - "hot" => { - "actions" => { - "rollover" => { - "max_docs" => "3" - } - } - } - } - }}} + subject.multi_receive([ + LogStash::Event.new("message" => "sample message here"), + LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }), + LogStash::Event.new("somevalue" => 100), + ]) - before do - expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) - put_policy(@es,ilm_policy_name, policy) - end + sleep(6) - it 'should not install the default policy if it is not used' do - subject.register - sleep(1) - expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) + subject.multi_receive([ + LogStash::Event.new("country" => "us"), + LogStash::Event.new("country" => "at"), + LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] }) + ]) + + @es.indices.refresh + + # Wait or fail until everything's indexed. + Stud::try(20.times) do + r = @es.search + expect(r["hits"]["total"]).to eq(6) + end + indexes_written = @es.search['hits']['hits'].each_with_object(Hash.new(0)) do |x, res| + index_written = x['_index'] + res[index_written] += 1 + end + expect(indexes_written.count).to eq(1) end end - context 'with ilm enabled' do - let (:ilm_enabled) { true } +end - it 'should write the write alias' do - expect(@es.indices.exists_alias(index: ilm_write_alias)).to be_falsey - subject.register - sleep(1) - expect(@es.indices.exists_alias(index: ilm_write_alias)).to be_truthy - end +shared_examples_for 'an ILM enabled Logstash' do - context 'with a policy with a maximum number of documents' do - let (:policy) { small_max_doc_policy } - let (:ilm_policy_name) { "few_docs"} - let (:settings) { super.merge("ilm_policy" => ilm_policy_name)} + context 'with a policy with a maximum number of documents' do + let (:policy) { small_max_doc_policy } + let (:ilm_policy_name) { "custom-policy"} + let (:settings) { super.merge("ilm_policy" => ilm_policy_name)} - it 'should rollover when the policy max docs is reached' do - put_policy(@es,ilm_policy_name, policy) - subject.register + it 'should rollover when the policy max docs is reached' do + put_policy(@es,ilm_policy_name, policy) + subject.register - subject.multi_receive([ - LogStash::Event.new("message" => "sample message here"), - LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }), - LogStash::Event.new("somevalue" => 100), - ]) + subject.multi_receive([ + LogStash::Event.new("message" => "sample message here"), + LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }), + LogStash::Event.new("somevalue" => 100), + ]) - sleep(6) + sleep(6) - subject.multi_receive([ - LogStash::Event.new("country" => "us"), - LogStash::Event.new("country" => "at"), - LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] }) - ]) + subject.multi_receive([ + LogStash::Event.new("country" => "us"), + LogStash::Event.new("country" => "at"), + LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] }) + ]) - sleep(6) + sleep(6) - subject.multi_receive([ - LogStash::Event.new("country" => "uk"), - LogStash::Event.new("country" => "fr"), - LogStash::Event.new("geoip" => { "location" => [ 0.1, 1.0 ] }) - ]) + subject.multi_receive([ + LogStash::Event.new("country" => "uk"), + LogStash::Event.new("country" => "fr"), + LogStash::Event.new("geoip" => { "location" => [ 0.1, 1.0 ] }) + ]) - @es.indices.refresh + @es.indices.refresh - # Wait or fail until everything's indexed. - Stud::try(20.times) do - r = @es.search - expect(r["hits"]["total"]).to eq(9) - end - indexes_written = @es.search['hits']['hits'].each_with_object(Hash.new(0)) do |x, res| - index_written = x['_index'] - res[index_written] += 1 - end - expect(indexes_written.count).to eq(3) - expect(indexes_written["#{ilm_write_alias}-000001"]).to eq(3) - expect(indexes_written["#{ilm_write_alias}-000002"]).to eq(3) - expect(indexes_written["#{ilm_write_alias}-000003"]).to eq(3) + # Wait or fail until everything's indexed. + Stud::try(20.times) do + r = @es.search + expect(r["hits"]["total"]).to eq(9) + end + indexes_written = @es.search['hits']['hits'].each_with_object(Hash.new(0)) do |x, res| + index_written = x['_index'] + res[index_written] += 1 end + expect(indexes_written.count).to eq(3) + expect(indexes_written["#{ilm_write_alias}-000001"]).to eq(3) + expect(indexes_written["#{ilm_write_alias}-000002"]).to eq(3) + expect(indexes_written["#{ilm_write_alias}-000003"]).to eq(3) end + end - context 'with a policy where the maximum number of documents is not reached' do - let (:policy) { large_max_doc_policy } - let (:ilm_policy_name) { "many_docs"} - let (:settings) { super.merge("ilm_policy" => ilm_policy_name)} + context 'with a policy where the maximum number of documents is not reached' do + let (:policy) { large_max_doc_policy } + let (:ilm_policy_name) { "custom-policy"} + let (:settings) { super.merge("ilm_policy" => ilm_policy_name)} - it 'should ingest into a single index when max docs is not reached' do - put_policy(@es,ilm_policy_name, policy) - subject.register + it 'should ingest into a single index when max docs is not reached' do + put_policy(@es,ilm_policy_name, policy) + subject.register - subject.multi_receive([ - LogStash::Event.new("message" => "sample message here"), - LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }), - LogStash::Event.new("somevalue" => 100), - ]) + subject.multi_receive([ + LogStash::Event.new("message" => "sample message here"), + LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }), + LogStash::Event.new("somevalue" => 100), + ]) - sleep(6) + sleep(6) - subject.multi_receive([ - LogStash::Event.new("country" => "us"), - LogStash::Event.new("country" => "at"), - LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] }) - ]) + subject.multi_receive([ + LogStash::Event.new("country" => "us"), + LogStash::Event.new("country" => "at"), + LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] }) + ]) - @es.indices.refresh + @es.indices.refresh - # Wait or fail until everything's indexed. - Stud::try(20.times) do - r = @es.search - expect(r["hits"]["total"]).to eq(6) - end - indexes_written = @es.search['hits']['hits'].each_with_object(Hash.new(0)) do |x, res| - index_written = x['_index'] - res[index_written] += 1 - end - expect(indexes_written.count).to eq(1) - expect(indexes_written["#{ilm_write_alias}-000001"]).to eq(6) + # Wait or fail until everything's indexed. + Stud::try(20.times) do + r = @es.search + expect(r["hits"]["total"]).to eq(6) + end + indexes_written = @es.search['hits']['hits'].each_with_object(Hash.new(0)) do |x, res| + index_written = x['_index'] + res[index_written] += 1 end + expect(indexes_written.count).to eq(1) + expect(indexes_written["#{ilm_write_alias}-000001"]).to eq(6) end end +end - context 'with ilm disabled' do - let (:ilm_enabled) { false } - it 'should not write the write alias' do - expect(@es.indices.exists_alias(index: ilm_write_alias)).to be_falsey - subject.register - sleep(1) - expect(@es.indices.exists_alias(index: ilm_write_alias)).to be_falsey - end +if ESHelper.es_version_satisfies?("<= 6.4") + describe 'Pre-ILM versions of Elasticsearch' do + it_behaves_like 'an Elasticsearch instance that does not support index lifecycle management' + end +end - it 'should not install the default policy' do - subject.register - sleep(1) - expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) - end +if ESHelper.es_version_satisfies?(">= 6.5") + describe 'OSS Elasticsearch', :skip => true do + it_behaves_like 'an Elasticsearch instance that does not support index lifecycle management' + end - context 'with an existing policy that will roll over' do - let (:policy) { small_max_doc_policy } - let (:ilm_policy_name) { "3_docs"} - let (:settings) { super.merge("ilm_policy" => ilm_policy_name)} + describe 'Elasticsearch has index lifecycle enabled' do #, :integration => true do + DEFAULT_INTERVAL = '600s' - it 'should index documents normally' do + require "logstash/outputs/elasticsearch" + let (:ilm_write_alias) { "the_write_alias" } + let (:index) { ilm_write_alias } + let (:ilm_enabled) { true } - subject.register - subject.multi_receive([ - LogStash::Event.new("message" => "sample message here"), - LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }), - LogStash::Event.new("somevalue" => 100), - ]) - - sleep(6) - - subject.multi_receive([ - LogStash::Event.new("country" => "us"), - LogStash::Event.new("country" => "at"), - LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] }) - ]) - - @es.indices.refresh - - # Wait or fail until everything's indexed. - Stud::try(20.times) do - r = @es.search - expect(r["hits"]["total"]).to eq(6) - end - indexes_written = @es.search['hits']['hits'].each_with_object(Hash.new(0)) do |x, res| - index_written = x['_index'] - res[index_written] += 1 - end - expect(indexes_written.count).to eq(1) - expect(indexes_written["#{index}"]).to eq(6) - end - end - end -end + let (:settings) { + { + "index" => index, + "ilm_enabled" => ilm_enabled, + "ilm_write_alias" => ilm_write_alias, + "manage_template" => true, + "template_name" => ilm_write_alias, + "template_overwrite" => true, + "hosts" => "#{get_host_port()}" + } + } + let (:policy) { small_max_doc_policy } -describe 'ES Does not support index lifecycle management' do - require "logstash/outputs/elasticsearch" - # let (:ilm_policy_name) {"three_and_done"} - let (:ilm_write_alias) { "the_write_alias" } - let (:index) { "ilm_write_alias" } - let (:ilm_enabled) { false } + let (:small_max_doc_policy) { + {"policy" => { + "phases"=> { + "hot" => { + "actions" => { + "rollover" => { + "max_docs" => "3" + } + } + } + } + }} + } - let (:settings) { - { - "index" => index, - "ilm_enabled" => ilm_enabled, - # "ilm_policy" => ilm_policy_name, - "ilm_write_alias" => ilm_write_alias, - "manage_template" => true, - "template_overwrite" => true, - "hosts" => "#{get_host_port()}" + let (:large_max_doc_policy) { + {"policy" => { + "phases"=> { + "hot" => { + "actions" => { + "rollover" => { + "max_docs" => "1000000" + } + } + } + } + }} } - } - subject { LogStash::Outputs::ElasticSearch.new(settings) } - context 'when ilm if disabled' do + subject { LogStash::Outputs::ElasticSearch.new(settings) } + before :each do + # Delete all templates first. require "elasticsearch" # Clean ES of data before we start. @es = get_client clean(@es) + # Set the poll interval for lifecycle management to be short so changes get picked up in time. + set_cluster_settings(@es, { + "persistent" => { + "indices.lifecycle.poll_interval" => "1s" + } + }) end after :each do - # set_cluster_settings(@es, @old_cluster_settings) + # Set poll interval back to default + set_cluster_settings(@es, { + "persistent" => { + "indices.lifecycle.poll_interval" => DEFAULT_INTERVAL + } + }) clean(@es) end - it 'should index documents normally' do - subject.register - subject.multi_receive([ - LogStash::Event.new("message" => "sample message here"), - LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }), - LogStash::Event.new("somevalue" => 100), - ]) + context 'with ilm enabled' do + let (:ilm_enabled) { true } - sleep(6) + context 'when using the default policy' do + it 'should install it if it is not present' do + expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) + subject.register + sleep(1) + expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.not_to raise_error + end + end - subject.multi_receive([ - LogStash::Event.new("country" => "us"), - LogStash::Event.new("country" => "at"), - LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] }) - ]) - @es.indices.refresh + context 'when not using the default policy' do + let (:ilm_policy_name) {"new_one"} + let (:settings) { super.merge("ilm_policy" => ilm_policy_name)} + let (:policy) {{ + "policy" => { + "phases"=> { + "hot" => { + "actions" => { + "rollover" => { + "max_docs" => "3" + } + } + } + } + }}} - # Wait or fail until everything's indexed. - Stud::try(20.times) do - r = @es.search - expect(r["hits"]["total"]).to eq(6) + before do + expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) + put_policy(@es,ilm_policy_name, policy) + end + + it 'should not install the default policy if it is not used' do + subject.register + sleep(1) + expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) + end end - indexes_written = @es.search['hits']['hits'].each_with_object(Hash.new(0)) do |x, res| - index_written = x['_index'] - res[index_written] += 1 + + context 'with the default template' do + + it 'should write the write alias' do + expect(@es.indices.exists_alias(index: ilm_write_alias)).to be_falsey + subject.register + sleep(1) + expect(@es.indices.exists_alias(index: ilm_write_alias)).to be_truthy + end + + it_behaves_like 'an ILM enabled Logstash' + end + + context 'with a custom template' do + let (:ilm_write_alias) { "custom" } + let (:index) { ilm_write_alias } + let (:template_name) { "custom" } + if ESHelper.es_version_satisfies?(">= 7.0") + let (:template) { "spec/fixtures/template-with-policy-es6x.json" } + else + let (:template) { "spec/fixtures/template-with-policy-es6x.json" } + end + let (:ilm_enabled) { true } + let (:ilm_policy_name) { "custom-policy" } + let (:settings) { super.merge("ilm_policy" => ilm_policy_name, "template" => template)} + + before :each do + put_policy(@es,ilm_policy_name, policy) + end + it 'should write the write alias' do + expect(@es.indices.exists_alias(index: ilm_write_alias)).to be_falsey + subject.register + sleep(1) + expect(@es.indices.exists_alias(index: ilm_write_alias)).to be_truthy + end + + it_behaves_like 'an ILM enabled Logstash' + end + end + + context 'with ilm disabled' do + let (:ilm_enabled) { false } + + it 'should not write the write alias' do + expect(@es.indices.exists_alias(index: ilm_write_alias)).to be_falsey + subject.register + sleep(1) + expect(@es.indices.exists_alias(index: ilm_write_alias)).to be_falsey + end + + it 'should not install the default policy' do + subject.register + sleep(1) + expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) + end + + context 'with an existing policy that will roll over' do + let (:policy) { small_max_doc_policy } + let (:ilm_policy_name) { "3_docs"} + let (:settings) { super.merge("ilm_policy" => ilm_policy_name)} + + it 'should index documents normally' do + + subject.register + subject.multi_receive([ + LogStash::Event.new("message" => "sample message here"), + LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }), + LogStash::Event.new("somevalue" => 100), + ]) + + sleep(6) + + subject.multi_receive([ + LogStash::Event.new("country" => "us"), + LogStash::Event.new("country" => "at"), + LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] }) + ]) + + @es.indices.refresh + + # Wait or fail until everything's indexed. + Stud::try(20.times) do + r = @es.search + expect(r["hits"]["total"]).to eq(6) + end + indexes_written = @es.search['hits']['hits'].each_with_object(Hash.new(0)) do |x, res| + index_written = x['_index'] + res[index_written] += 1 + end + expect(indexes_written.count).to eq(1) + expect(indexes_written["#{index}"]).to eq(6) + end end - expect(indexes_written.count).to eq(1) - expect(indexes_written["#{index}"]).to eq(6) end end end \ No newline at end of file From 37b1db049fb4d8f034d338a41df3f31301482c20 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Wed, 7 Nov 2018 10:30:42 -0500 Subject: [PATCH 06/19] Update _ilm endpoint to be _ilm/policy --- ci/run.sh | 1 + .../elasticsearch/default-ilm-policy.json | 2 +- .../outputs/elasticsearch/http_client.rb | 6 +-- spec/integration/outputs/ilm_spec.rb | 16 +++--- spec/integration/outputs/parent_spec.rb | 2 +- spec/integration/outputs/routing_spec.rb | 50 +++++++++---------- .../api/actions/delete_ilm_policy.rb | 4 +- .../api/actions/get_ilm_policy.rb | 2 +- .../api/actions/put_ilm_policy.rb | 2 +- 9 files changed, 43 insertions(+), 42 deletions(-) diff --git a/ci/run.sh b/ci/run.sh index f30409d99..633af20a8 100755 --- a/ci/run.sh +++ b/ci/run.sh @@ -173,6 +173,7 @@ else start_es bundle exec rspec -fd $extra_tag_args --tag update_tests:painless --tag update_tests:groovy --tag es_version:$es_distribution_version $spec_path ;; + 5.*) setup_es https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ES_VERSION}.tar.gz es_distribution_version=$(get_es_distribution_version) diff --git a/lib/logstash/outputs/elasticsearch/default-ilm-policy.json b/lib/logstash/outputs/elasticsearch/default-ilm-policy.json index 4ba3b3f7f..f2ac4da06 100644 --- a/lib/logstash/outputs/elasticsearch/default-ilm-policy.json +++ b/lib/logstash/outputs/elasticsearch/default-ilm-policy.json @@ -4,7 +4,7 @@ "hot" : { "actions" : { "rollover" : { - "max_size" : "25gb", + "max_size" : "50gb", "max_age":"30d" } } diff --git a/lib/logstash/outputs/elasticsearch/http_client.rb b/lib/logstash/outputs/elasticsearch/http_client.rb index d911ac3d6..c511c7bb3 100644 --- a/lib/logstash/outputs/elasticsearch/http_client.rb +++ b/lib/logstash/outputs/elasticsearch/http_client.rb @@ -366,15 +366,15 @@ def write_alias_put(alias_name, alias_definition) end def get_ilm_endpoint - @pool.get("/_ilm") + @pool.get("/_ilm/policy") end def ilm_policy_exists?(name) - exists?("/_ilm/#{name}", true) + exists?("/_ilm/policy/#{name}", true) end def ilm_policy_put(name, policy) - path = "_ilm/#{name}" + path = "_ilm/policy/#{name}" logger.info("Installing ILM policy #{policy} to #{path}") @pool.put(path, nil, LogStash::Json.dump(policy)) end diff --git a/spec/integration/outputs/ilm_spec.rb b/spec/integration/outputs/ilm_spec.rb index d7ab1eee8..a3932df8a 100644 --- a/spec/integration/outputs/ilm_spec.rb +++ b/spec/integration/outputs/ilm_spec.rb @@ -166,18 +166,18 @@ end -if ESHelper.es_version_satisfies?("<= 6.4") - describe 'Pre-ILM versions of Elasticsearch' do +if ESHelper.es_version_satisfies?("<= 6.5") + describe 'Pre-ILM versions of Elasticsearch', :integration => true do it_behaves_like 'an Elasticsearch instance that does not support index lifecycle management' end end -if ESHelper.es_version_satisfies?(">= 6.5") - describe 'OSS Elasticsearch', :skip => true do - it_behaves_like 'an Elasticsearch instance that does not support index lifecycle management' - end +if ESHelper.es_version_satisfies?(">= 6.6") + # describe 'OSS Elasticsearch', :integration => true do + # it_behaves_like 'an Elasticsearch instance that does not support index lifecycle management' + # end - describe 'Elasticsearch has index lifecycle enabled' do #, :integration => true do + describe 'Elasticsearch has index lifecycle management enabled', :integration => true do DEFAULT_INTERVAL = '600s' require "logstash/outputs/elasticsearch" @@ -314,7 +314,7 @@ let (:index) { ilm_write_alias } let (:template_name) { "custom" } if ESHelper.es_version_satisfies?(">= 7.0") - let (:template) { "spec/fixtures/template-with-policy-es6x.json" } + let (:template) { "spec/fixtures/template-with-policy-es7x.json" } else let (:template) { "spec/fixtures/template-with-policy-es6x.json" } end diff --git a/spec/integration/outputs/parent_spec.rb b/spec/integration/outputs/parent_spec.rb index 117ba1b8e..c838b0e71 100644 --- a/spec/integration/outputs/parent_spec.rb +++ b/spec/integration/outputs/parent_spec.rb @@ -2,7 +2,7 @@ require "logstash/outputs/elasticsearch" if ESHelper.es_version_satisfies?("<= 5.x") - context "when using elasticsearch 5.x and before", :integration => true do + context "when using elasticsearch 5.x and before", :integration => "changethis" do shared_examples "a type based parent indexer" do let(:index) { 10.times.collect { rand(10).to_s }.join("") } let(:type) { 10.times.collect { rand(10).to_s }.join("") } diff --git a/spec/integration/outputs/routing_spec.rb b/spec/integration/outputs/routing_spec.rb index d4e23cba6..a09f91cb8 100644 --- a/spec/integration/outputs/routing_spec.rb +++ b/spec/integration/outputs/routing_spec.rb @@ -34,28 +34,28 @@ end end -describe "(http protocol) index events with static routing", :integration => true do - it_behaves_like 'a routing indexer' do - let(:routing) { "test" } - let(:config) { - { - "hosts" => get_host_port, - "index" => index, - "routing" => routing - } - } - end -end - -describe "(http_protocol) index events with fieldref in routing value", :integration => true do - it_behaves_like 'a routing indexer' do - let(:routing) { "test" } - let(:config) { - { - "hosts" => get_host_port, - "index" => index, - "routing" => "%{message}" - } - } - end -end +# describe "(http protocol) index events with static routing", :integration => true do +# it_behaves_like 'a routing indexer' do +# let(:routing) { "test" } +# let(:config) { +# { +# "hosts" => get_host_port, +# "index" => index, +# "routing" => routing +# } +# } +# end +# end +# +# describe "(http_protocol) index events with fieldref in routing value", :integration => true do +# it_behaves_like 'a routing indexer' do +# let(:routing) { "test" } +# let(:config) { +# { +# "hosts" => get_host_port, +# "index" => index, +# "routing" => "%{message}" +# } +# } +# end +# end diff --git a/spec/support/elasticsearch/api/actions/delete_ilm_policy.rb b/spec/support/elasticsearch/api/actions/delete_ilm_policy.rb index 05c529f46..83c66f3c7 100644 --- a/spec/support/elasticsearch/api/actions/delete_ilm_policy.rb +++ b/spec/support/elasticsearch/api/actions/delete_ilm_policy.rb @@ -9,11 +9,11 @@ module Actions # Update the password of the specified user def delete_ilm_policy(arguments={}) method = HTTP_DELETE - path = Utils.__pathify '_ilm/', + path = Utils.__pathify '_ilm/policy/', Utils.__escape(arguments[:name]) params = {} perform_request(method, path, params, nil).body end end end -end \ No newline at end of file +end diff --git a/spec/support/elasticsearch/api/actions/get_ilm_policy.rb b/spec/support/elasticsearch/api/actions/get_ilm_policy.rb index eb993bb61..accf98466 100644 --- a/spec/support/elasticsearch/api/actions/get_ilm_policy.rb +++ b/spec/support/elasticsearch/api/actions/get_ilm_policy.rb @@ -9,7 +9,7 @@ module Actions # Retrieve the list of index lifecycle management policies def get_ilm_policy(arguments={}) method = HTTP_GET - path = Utils.__pathify '_ilm', Utils.__escape(arguments[:name]) + path = Utils.__pathify '_ilm/policy', Utils.__escape(arguments[:name]) params = {} perform_request(method, path, params, nil).body end diff --git a/spec/support/elasticsearch/api/actions/put_ilm_policy.rb b/spec/support/elasticsearch/api/actions/put_ilm_policy.rb index 111cbaa20..57e8305b7 100644 --- a/spec/support/elasticsearch/api/actions/put_ilm_policy.rb +++ b/spec/support/elasticsearch/api/actions/put_ilm_policy.rb @@ -13,7 +13,7 @@ def put_ilm_policy(arguments={}) raise ArgumentError, "Required argument 'name' missing" unless arguments[:name] raise ArgumentError, "Required argument 'body' missing" unless arguments[:body] method = HTTP_PUT - path = Utils.__pathify '_ilm', Utils.__escape(arguments[:name]) + path = Utils.__pathify '_ilm/policy/', Utils.__escape(arguments[:name]) params = Utils.__validate_and_extract_params arguments From 9ef1c32f6d2351124b9919926fbae66d65d3a54a Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Fri, 9 Nov 2018 09:47:58 -0500 Subject: [PATCH 07/19] Use _xpack endpoint to verify ilm enablement status on Elasticsearch --- ci/run.sh | 6 ++++++ lib/logstash/outputs/elasticsearch/common.rb | 14 +++++++++++--- lib/logstash/outputs/elasticsearch/http_client.rb | 6 +++++- spec/integration/outputs/ilm_spec.rb | 12 ++++++------ 4 files changed, 28 insertions(+), 10 deletions(-) diff --git a/ci/run.sh b/ci/run.sh index 633af20a8..7f4d80db1 100755 --- a/ci/run.sh +++ b/ci/run.sh @@ -131,6 +131,12 @@ else extra_tag_args="--tag secure_integration" fi + if [[ "$DISTRIBUTION" == "oss" ]]; then + extra_tag_args="$extra_tag_args --tag distribution:oss --tag ~distribution:xpack" + elif [[ "$DISTRIBUTION" == "default" ]]; then + extra_tag_args="$extra_tag_args --tag ~distribution:oss --tag distribution:xpack" + fi + case "$ES_VERSION" in LATEST-SNAPSHOT-*) split_latest=${ES_VERSION##*-} diff --git a/lib/logstash/outputs/elasticsearch/common.rb b/lib/logstash/outputs/elasticsearch/common.rb index c72e45e7d..7597e7ab6 100644 --- a/lib/logstash/outputs/elasticsearch/common.rb +++ b/lib/logstash/outputs/elasticsearch/common.rb @@ -377,17 +377,25 @@ def verify_ilm return true unless ilm_enabled? begin verify_ilm_config + # For elasticsearch versions without ilm enablement, the _ilm endpoint will return a 400 bad request - the # endpoint is interpreted as an index, and will return a bad request error as that is an illegal format for # an index. - client.get_ilm_endpoint + xpack = client.get_xpack_info + features = xpack["features"] + ilm = features["ilm"] unless features.nil? + raise LogStash::ConfigurationError, "Index Lifecycle management is enabled in logstash, but not installed on your Elasticsearch cluster" if features.nil? || ilm.nil? + raise LogStash::ConfigurationError, "Index Lifecycle management is enabled in logstash, but not available in your Elasticsearch cluster" unless ilm['available'] + raise LogStash::ConfigurationError, "Index Lifecycle management is enabled in logstash, but not enabled in your Elasticsearch cluster" unless ilm['enabled'] true rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e + # Check xpack endpoint: If no xpack endpoint, then this version of Elasticsearch is not compatible if e.response_code == 404 - true + false + raise LogStash::ConfigurationError, "Index Lifecycle management is enabled in logstash, but not installed on your Elasticsearch cluster" elsif e.response_code == 400 false - raise LogStash::ConfigurationError, "Index Lifecycle Management feature is enabled, but Elasticsearch instance does not have ilm capability, #{e}" + raise LogStash::ConfigurationError, "Index Lifecycle management is enabled in logstash, but not installed on your Elasticsearch cluster" else raise e end diff --git a/lib/logstash/outputs/elasticsearch/http_client.rb b/lib/logstash/outputs/elasticsearch/http_client.rb index c511c7bb3..fd3b8454e 100644 --- a/lib/logstash/outputs/elasticsearch/http_client.rb +++ b/lib/logstash/outputs/elasticsearch/http_client.rb @@ -365,7 +365,11 @@ def write_alias_put(alias_name, alias_definition) @pool.put(alias_name, nil, LogStash::Json.dump(alias_definition)) end - def get_ilm_endpoint + def get_xpack_info + get("/_xpack") + end + + def get_ilm_endpoint @pool.get("/_ilm/policy") end diff --git a/spec/integration/outputs/ilm_spec.rb b/spec/integration/outputs/ilm_spec.rb index a3932df8a..a47a8c369 100644 --- a/spec/integration/outputs/ilm_spec.rb +++ b/spec/integration/outputs/ilm_spec.rb @@ -25,7 +25,7 @@ subject { LogStash::Outputs::ElasticSearch.new(settings) } - context 'when ilm is enabled' do + context 'when ilm is enabled in Logstash' do let (:ilm_enabled) { true } it 'should raise a configuration error' do @@ -39,7 +39,7 @@ end end - context 'when ilm is disabled' do + context 'when ilm is disabled in Logstash' do it 'should index documents normally' do subject.register @@ -173,11 +173,11 @@ end if ESHelper.es_version_satisfies?(">= 6.6") - # describe 'OSS Elasticsearch', :integration => true do - # it_behaves_like 'an Elasticsearch instance that does not support index lifecycle management' - # end + describe 'OSS Elasticsearch', :distribution => 'oss', :integration => true do + it_behaves_like 'an Elasticsearch instance that does not support index lifecycle management' + end - describe 'Elasticsearch has index lifecycle management enabled', :integration => true do + describe 'Elasticsearch has index lifecycle management enabled', :distribution => 'xpack', :integration => true do DEFAULT_INTERVAL = '600s' require "logstash/outputs/elasticsearch" From 2b9496bace71f8629c140367de44b2c44a427833 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Mon, 12 Nov 2018 09:55:17 -0500 Subject: [PATCH 08/19] Update travis.yml to include more integration tests Add tests against 6.6 snapshot (first Elasticsearch version to include ILM) and update test against 7.0 to retrieve artifact, rather than attempt to build master. --- spec/integration/outputs/ilm_spec.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/spec/integration/outputs/ilm_spec.rb b/spec/integration/outputs/ilm_spec.rb index a47a8c369..37f9bd017 100644 --- a/spec/integration/outputs/ilm_spec.rb +++ b/spec/integration/outputs/ilm_spec.rb @@ -32,6 +32,7 @@ expect do begin subject.register + sleep(1) ensure subject.stop_template_installer end From 2aff4accdacf7f814ed76cc0ed4429a7906ca2a0 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Mon, 12 Nov 2018 10:50:48 -0500 Subject: [PATCH 09/19] Add missing test templates --- spec/fixtures/template-with-policy-es6x.json | 48 ++++++++++++++++++++ spec/fixtures/template-with-policy-es7x.json | 48 ++++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 spec/fixtures/template-with-policy-es6x.json create mode 100644 spec/fixtures/template-with-policy-es7x.json diff --git a/spec/fixtures/template-with-policy-es6x.json b/spec/fixtures/template-with-policy-es6x.json new file mode 100644 index 000000000..4566fa852 --- /dev/null +++ b/spec/fixtures/template-with-policy-es6x.json @@ -0,0 +1,48 @@ +{ + "template" : "custom-*", + "version" : 60001, + "settings" : { + "index.refresh_interval" : "1s", + "number_of_shards": 1, + "index.lifecycle.name": "custom-policy", + "index.lifecycle.rollover_alias": "custom" + }, + "mappings" : { + "_default_" : { + "dynamic_templates" : [ { + "message_field" : { + "path_match" : "message", + "match_mapping_type" : "string", + "mapping" : { + "type" : "text", + "norms" : false + } + } + }, { + "string_fields" : { + "match" : "*", + "match_mapping_type" : "string", + "mapping" : { + "type" : "text", "norms" : false, + "fields" : { + "keyword" : { "type": "keyword", "ignore_above": 256 } + } + } + } + } ], + "properties" : { + "@timestamp": { "type": "date"}, + "@version": { "type": "keyword"}, + "geoip" : { + "dynamic": true, + "properties" : { + "ip": { "type": "ip" }, + "location" : { "type" : "geo_point" }, + "latitude" : { "type" : "half_float" }, + "longitude" : { "type" : "half_float" } + } + } + } + } + } +} diff --git a/spec/fixtures/template-with-policy-es7x.json b/spec/fixtures/template-with-policy-es7x.json new file mode 100644 index 000000000..c8a0ab192 --- /dev/null +++ b/spec/fixtures/template-with-policy-es7x.json @@ -0,0 +1,48 @@ +{ + "template" : "custom-*", + "version" : 60001, + "settings" : { + "index.refresh_interval" : "1s", + "number_of_shards": 1, + "index.lifecycle.name": "custom-policy", + "index.lifecycle.rollover_alias": "custom" + }, + "mappings" : { + "_doc" : { + "dynamic_templates" : [ { + "message_field" : { + "path_match" : "message", + "match_mapping_type" : "string", + "mapping" : { + "type" : "text", + "norms" : false + } + } + }, { + "string_fields" : { + "match" : "*", + "match_mapping_type" : "string", + "mapping" : { + "type" : "text", "norms" : false, + "fields" : { + "keyword" : { "type": "keyword", "ignore_above": 256 } + } + } + } + } ], + "properties" : { + "@timestamp": { "type": "date"}, + "@version": { "type": "keyword"}, + "geoip" : { + "dynamic": true, + "properties" : { + "ip": { "type": "ip" }, + "location" : { "type" : "geo_point" }, + "latitude" : { "type" : "half_float" }, + "longitude" : { "type" : "half_float" } + } + } + } + } + } +} From 3b0544357413b759fcf024941c65fbd8a5377db3 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Tue, 20 Nov 2018 11:02:59 -0500 Subject: [PATCH 10/19] Integration test improvements Add more tests to check correct creation of write alias and templates Tests for default settings --- spec/es_spec_helper.rb | 1 + spec/integration/outputs/ilm_spec.rb | 111 ++++++++++++++---- .../elasticsearch/api/actions/get_alias.rb | 18 +++ 3 files changed, 109 insertions(+), 21 deletions(-) create mode 100644 spec/support/elasticsearch/api/actions/get_alias.rb diff --git a/spec/es_spec_helper.rb b/spec/es_spec_helper.rb index 58f9cbe30..14738f91c 100644 --- a/spec/es_spec_helper.rb +++ b/spec/es_spec_helper.rb @@ -2,6 +2,7 @@ require 'manticore' require 'elasticsearch' require_relative "support/elasticsearch/api/actions/delete_ilm_policy" +require_relative "support/elasticsearch/api/actions/get_alias" require_relative "support/elasticsearch/api/actions/get_ilm_policy" require_relative "support/elasticsearch/api/actions/put_ilm_policy" diff --git a/spec/integration/outputs/ilm_spec.rb b/spec/integration/outputs/ilm_spec.rb index 37f9bd017..62e4d12a6 100644 --- a/spec/integration/outputs/ilm_spec.rb +++ b/spec/integration/outputs/ilm_spec.rb @@ -120,9 +120,9 @@ res[index_written] += 1 end expect(indexes_written.count).to eq(3) - expect(indexes_written["#{ilm_write_alias}-000001"]).to eq(3) - expect(indexes_written["#{ilm_write_alias}-000002"]).to eq(3) - expect(indexes_written["#{ilm_write_alias}-000003"]).to eq(3) + expect(indexes_written["#{expected_index}-000001"]).to eq(3) + expect(indexes_written["#{expected_index}-000002"]).to eq(3) + expect(indexes_written["#{expected_index}-000003"]).to eq(3) end end @@ -161,7 +161,7 @@ res[index_written] += 1 end expect(indexes_written.count).to eq(1) - expect(indexes_written["#{ilm_write_alias}-000001"]).to eq(6) + expect(indexes_written["#{expected_index}-000001"]).to eq(6) end end end @@ -182,18 +182,11 @@ DEFAULT_INTERVAL = '600s' require "logstash/outputs/elasticsearch" - let (:ilm_write_alias) { "the_write_alias" } - let (:index) { ilm_write_alias } let (:ilm_enabled) { true } let (:settings) { { - "index" => index, "ilm_enabled" => ilm_enabled, - "ilm_write_alias" => ilm_write_alias, - "manage_template" => true, - "template_name" => ilm_write_alias, - "template_overwrite" => true, "hosts" => "#{get_host_port()}" } } @@ -260,6 +253,7 @@ context 'with ilm enabled' do let (:ilm_enabled) { true } + context 'when using the default policy' do it 'should install it if it is not present' do expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) @@ -267,8 +261,49 @@ sleep(1) expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.not_to raise_error end - end + it 'should create the default write alias' do + + expect(@es.indices.exists_alias(index: "logstash")).to be_falsey + subject.register + sleep(1) + expect(@es.indices.exists_alias(index: "logstash")).to be_truthy + expect(@es.get_alias(name: "logstash")).to include("logstash-000001") + end + + it 'should ingest into a single index' do + subject.register + + subject.multi_receive([ + LogStash::Event.new("message" => "sample message here"), + LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }), + LogStash::Event.new("somevalue" => 100), + ]) + + sleep(6) + + subject.multi_receive([ + LogStash::Event.new("country" => "us"), + LogStash::Event.new("country" => "at"), + LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] }) + ]) + + @es.indices.refresh + + # Wait or fail until everything's indexed. + Stud::try(20.times) do + r = @es.search + expect(r["hits"]["total"]).to eq(6) + end + indexes_written = @es.search['hits']['hits'].each_with_object(Hash.new(0)) do |x, res| + index_written = x['_index'] + res[index_written] += 1 + end + + expect(indexes_written.count).to eq(1) + expect(indexes_written["logstash-000001"]).to eq(6) + end + end context 'when not using the default policy' do let (:ilm_policy_name) {"new_one"} @@ -299,12 +334,22 @@ end context 'with the default template' do + let(:expected_index) { "logstash" } - it 'should write the write alias' do - expect(@es.indices.exists_alias(index: ilm_write_alias)).to be_falsey + it 'should create the write alias' do + expect(@es.indices.exists_alias(index: expected_index)).to be_falsey subject.register sleep(1) - expect(@es.indices.exists_alias(index: ilm_write_alias)).to be_truthy + expect(@es.indices.exists_alias(index: expected_index)).to be_truthy + expect(@es.get_alias(name: expected_index)).to include("#{expected_index}-000001") + end + + it 'should write the ILM settings into the template' do + subject.register + sleep(1) + expect(@es.indices.get_template(name: "logstash")["logstash"]["index_patterns"]).to eq(["logstash-*"]) + expect(@es.indices.get_template(name: "logstash")["logstash"]["settings"]['index']['lifecycle']['name']).to eq("logstash-policy") + expect(@es.indices.get_template(name: "logstash")["logstash"]["settings"]['index']['lifecycle']['rollover_alias']).to eq("logstash") end it_behaves_like 'an ILM enabled Logstash' @@ -313,7 +358,13 @@ context 'with a custom template' do let (:ilm_write_alias) { "custom" } let (:index) { ilm_write_alias } + let(:expected_index) { index } + let (:settings) { super.merge("ilm_policy" => ilm_policy_name, + "template" => template, + "template_name" => template_name, + "ilm_write_alias" => ilm_write_alias)} let (:template_name) { "custom" } + if ESHelper.es_version_satisfies?(">= 7.0") let (:template) { "spec/fixtures/template-with-policy-es7x.json" } else @@ -321,16 +372,25 @@ end let (:ilm_enabled) { true } let (:ilm_policy_name) { "custom-policy" } - let (:settings) { super.merge("ilm_policy" => ilm_policy_name, "template" => template)} + before :each do put_policy(@es,ilm_policy_name, policy) end - it 'should write the write alias' do + it 'should create the write alias' do expect(@es.indices.exists_alias(index: ilm_write_alias)).to be_falsey subject.register sleep(1) expect(@es.indices.exists_alias(index: ilm_write_alias)).to be_truthy + expect(@es.get_alias(name: ilm_write_alias)).to include("#{ilm_write_alias}-000001") + end + + it 'should write the ILM settings into the template' do + subject.register + sleep(1) + expect(@es.indices.get_template(name: template_name)[template_name]["index_patterns"]).to eq(["#{ilm_write_alias}-*"]) + expect(@es.indices.get_template(name: template_name)[template_name]["settings"]['index']['lifecycle']['name']).to eq(ilm_policy_name) + expect(@es.indices.get_template(name: template_name)[template_name]["settings"]['index']['lifecycle']['rollover_alias']).to eq(ilm_write_alias) end it_behaves_like 'an ILM enabled Logstash' @@ -340,11 +400,11 @@ context 'with ilm disabled' do let (:ilm_enabled) { false } - it 'should not write the write alias' do - expect(@es.indices.exists_alias(index: ilm_write_alias)).to be_falsey + it 'should not create a write alias' do + expect(@es.get_alias).to be_empty subject.register sleep(1) - expect(@es.indices.exists_alias(index: ilm_write_alias)).to be_falsey + expect(@es.get_alias).to be_empty end it 'should not install the default policy' do @@ -353,6 +413,15 @@ expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) end + it 'should write the ILM settings into the template' do + subject.register + sleep(1) + expect(@es.indices.get_template(name: "logstash")["logstash"]["index_patterns"]).to eq(["logstash-*"]) + expect(@es.indices.get_template(name: "logstash")["logstash"]["settings"]['index']['lifecycle']).to be_nil + + end + + context 'with an existing policy that will roll over' do let (:policy) { small_max_doc_policy } let (:ilm_policy_name) { "3_docs"} @@ -387,7 +456,7 @@ res[index_written] += 1 end expect(indexes_written.count).to eq(1) - expect(indexes_written["#{index}"]).to eq(6) + expect(indexes_written.values.first).to eq(6) end end end diff --git a/spec/support/elasticsearch/api/actions/get_alias.rb b/spec/support/elasticsearch/api/actions/get_alias.rb new file mode 100644 index 000000000..ef4ebbd4f --- /dev/null +++ b/spec/support/elasticsearch/api/actions/get_alias.rb @@ -0,0 +1,18 @@ +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. + +module Elasticsearch + module API + module Actions + + # Retrieve the list of index lifecycle management policies + def get_alias(arguments={}) + method = HTTP_GET + path = Utils.__pathify '_alias', Utils.__escape(arguments[:name]) + params = {} + perform_request(method, path, params, nil).body + end + end + end +end \ No newline at end of file From 65f8ffcfad9d2c50fdf82cde13593ff12c06a8e5 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Wed, 21 Nov 2018 14:19:46 -0500 Subject: [PATCH 11/19] Allow date math in pattern --- lib/logstash/outputs/elasticsearch/common.rb | 4 +- .../outputs/elasticsearch/common_configs.rb | 4 +- .../outputs/elasticsearch/http_client.rb | 2 +- .../http_client/manticore_adapter.rb | 6 +- spec/es_spec_helper.rb | 4 ++ spec/integration/outputs/ilm_spec.rb | 57 ++++++++++++++++--- 6 files changed, 61 insertions(+), 16 deletions(-) diff --git a/lib/logstash/outputs/elasticsearch/common.rb b/lib/logstash/outputs/elasticsearch/common.rb index 7597e7ab6..e71107d83 100644 --- a/lib/logstash/outputs/elasticsearch/common.rb +++ b/lib/logstash/outputs/elasticsearch/common.rb @@ -433,7 +433,7 @@ def maybe_create_write_alias end def write_alias_target - "#{ilm_write_alias}-#{ilm_pattern}" + "<#{ilm_write_alias}-#{ilm_pattern}>" end def write_alias_payload @@ -446,6 +446,8 @@ def write_alias_payload } end + + def policy_payload policy_path = ::File.expand_path(ILM_POLICY_PATH, ::File.dirname(__FILE__)) LogStash::Json.load(::IO.read(policy_path)) diff --git a/lib/logstash/outputs/elasticsearch/common_configs.rb b/lib/logstash/outputs/elasticsearch/common_configs.rb index 4d5c9cbc7..0e4bbce1b 100644 --- a/lib/logstash/outputs/elasticsearch/common_configs.rb +++ b/lib/logstash/outputs/elasticsearch/common_configs.rb @@ -149,8 +149,8 @@ def self.included(mod) # Write alias used for indexing data. If write alias doesn't exist, Logstash will create it and map it to the relevant index mod.config :ilm_write_alias, :validate => :string, :default => 'logstash' - # appends “000001” by default for new index creation, subsequent rollover indices will increment based on this pattern i.e. “000002” - mod.config :ilm_pattern, :validate => :string, :default => '000001' + # appends “{now/d}-000001” by default for new index creation, subsequent rollover indices will increment based on this pattern i.e. “000002” + mod.config :ilm_pattern, :validate => :string, :default => '{now/d}-000001' # ILM policy to use, if undefined the default policy will be used. mod.config :ilm_policy, :validate => :string, :default => 'logstash-policy' diff --git a/lib/logstash/outputs/elasticsearch/http_client.rb b/lib/logstash/outputs/elasticsearch/http_client.rb index fd3b8454e..806960a6a 100644 --- a/lib/logstash/outputs/elasticsearch/http_client.rb +++ b/lib/logstash/outputs/elasticsearch/http_client.rb @@ -362,7 +362,7 @@ def write_alias_exists?(name) # Create a new write alias def write_alias_put(alias_name, alias_definition) logger.info("Creating write alias #{alias_name}") - @pool.put(alias_name, nil, LogStash::Json.dump(alias_definition)) + @pool.put(CGI::escape(alias_name), nil, LogStash::Json.dump(alias_definition)) end def get_xpack_info diff --git a/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb b/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb index 6e12db126..96e8ae671 100644 --- a/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb +++ b/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb @@ -103,9 +103,9 @@ def format_url(url, path_and_query=nil) end request_uri.query = new_query_parts.join("&") unless new_query_parts.empty? - - request_uri.path = "#{request_uri.path}/#{parsed_path_and_query.path}".gsub(/\/{2,}/, "/") - + + request_uri.path = "#{request_uri.path}/#{parsed_path_and_query.raw_path}".gsub(/\/{2,}/, "/") + request_uri end diff --git a/spec/es_spec_helper.rb b/spec/es_spec_helper.rb index 14738f91c..732a69a62 100644 --- a/spec/es_spec_helper.rb +++ b/spec/es_spec_helper.rb @@ -25,6 +25,10 @@ def doc_type end end + def todays_date + Time.now.strftime("%Y.%m.%d") + end + def mapping_name if ESHelper.es_version_satisfies?(">=7") "_doc" diff --git a/spec/integration/outputs/ilm_spec.rb b/spec/integration/outputs/ilm_spec.rb index 62e4d12a6..a7ab9b0af 100644 --- a/spec/integration/outputs/ilm_spec.rb +++ b/spec/integration/outputs/ilm_spec.rb @@ -120,9 +120,9 @@ res[index_written] += 1 end expect(indexes_written.count).to eq(3) - expect(indexes_written["#{expected_index}-000001"]).to eq(3) - expect(indexes_written["#{expected_index}-000002"]).to eq(3) - expect(indexes_written["#{expected_index}-000003"]).to eq(3) + expect(indexes_written["#{expected_index}-#{todays_date}-000001"]).to eq(3) + expect(indexes_written["#{expected_index}-#{todays_date}-000002"]).to eq(3) + expect(indexes_written["#{expected_index}-#{todays_date}-000003"]).to eq(3) end end @@ -161,7 +161,7 @@ res[index_written] += 1 end expect(indexes_written.count).to eq(1) - expect(indexes_written["#{expected_index}-000001"]).to eq(6) + expect(indexes_written["#{expected_index}-#{todays_date}-000001"]).to eq(6) end end end @@ -255,6 +255,18 @@ context 'when using the default policy' do + context 'with a custom pattern' do + let (:settings) { super.merge("ilm_pattern" => "000001")} + it 'should create a write alias' do + expect(@es.indices.exists_alias(index: "logstash")).to be_falsey + subject.register + sleep(1) + expect(@es.indices.exists_alias(index: "logstash")).to be_truthy + expect(@es.get_alias(name: "logstash")).to include("logstash-000001") + end + end + + it 'should install it if it is not present' do expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) subject.register @@ -263,14 +275,14 @@ end it 'should create the default write alias' do - expect(@es.indices.exists_alias(index: "logstash")).to be_falsey subject.register sleep(1) expect(@es.indices.exists_alias(index: "logstash")).to be_truthy - expect(@es.get_alias(name: "logstash")).to include("logstash-000001") + expect(@es.get_alias(name: "logstash")).to include("logstash-#{todays_date}-000001") end + it 'should ingest into a single index' do subject.register @@ -301,7 +313,7 @@ end expect(indexes_written.count).to eq(1) - expect(indexes_written["logstash-000001"]).to eq(6) + expect(indexes_written["logstash-#{todays_date}-000001"]).to eq(6) end end @@ -333,6 +345,33 @@ end end + context 'when using a time based policy' do + let (:ilm_policy_name) {"new_one"} + let (:settings) { super.merge("ilm_policy" => ilm_policy_name)} + let (:policy) {{ + "policy" => { + "phases"=> { + "hot" => { + "actions" => { + "rollover" => { + "max_age" => "1d" + } + } + } + } + }}} + + before do + expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) + put_policy(@es,ilm_policy_name, policy) + end + + it 'should not install the default policy if it is not used' do + subject.register + sleep(1) + expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) + end + end context 'with the default template' do let(:expected_index) { "logstash" } @@ -341,7 +380,7 @@ subject.register sleep(1) expect(@es.indices.exists_alias(index: expected_index)).to be_truthy - expect(@es.get_alias(name: expected_index)).to include("#{expected_index}-000001") + expect(@es.get_alias(name: expected_index)).to include("#{expected_index}-#{todays_date}-000001") end it 'should write the ILM settings into the template' do @@ -382,7 +421,7 @@ subject.register sleep(1) expect(@es.indices.exists_alias(index: ilm_write_alias)).to be_truthy - expect(@es.get_alias(name: ilm_write_alias)).to include("#{ilm_write_alias}-000001") + expect(@es.get_alias(name: ilm_write_alias)).to include("#{ilm_write_alias}-#{todays_date}-000001") end it 'should write the ILM settings into the template' do From 21253a36c064df0939d026ecdba91b4653626c55 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Mon, 3 Dec 2018 18:02:53 -0500 Subject: [PATCH 12/19] Improve logging and error handling --- lib/logstash/outputs/elasticsearch/common.rb | 9 +++---- .../outputs/elasticsearch/http_client.rb | 16 ++++++++++++- .../outputs/elasticsearch/template_manager.rb | 4 ++++ spec/es_spec_helper.rb | 12 ++++++++++ spec/fixtures/template-with-policy-es7x.json | 4 +--- spec/integration/outputs/ilm_spec.rb | 14 +++++++++++ .../elasticsearch/api/actions/put_alias.rb | 24 +++++++++++++++++++ .../api/actions/put_ilm_policy.rb | 4 ++-- 8 files changed, 75 insertions(+), 12 deletions(-) create mode 100644 spec/support/elasticsearch/api/actions/put_alias.rb diff --git a/lib/logstash/outputs/elasticsearch/common.rb b/lib/logstash/outputs/elasticsearch/common.rb index e71107d83..6aa629cb1 100644 --- a/lib/logstash/outputs/elasticsearch/common.rb +++ b/lib/logstash/outputs/elasticsearch/common.rb @@ -403,10 +403,9 @@ def verify_ilm end def verify_ilm_config - # Default index name is logstash-XXXX and the default write alias is logstash - should we just change overwrite the default index name here? - @index = @ilm_write_alias if @index == LogStash::Outputs::ElasticSearch::CommonConfigs::DEFAULT_INDEX_NAME - raise LogStash::ConfigurationError, "ILM configuration error: The index name #{@index} does not match the write alias #{@ilm_write_alias}" if @ilm_write_alias != @index - raise LogStash::ConfigurationError, "ILM configuration error: The template name #{@template_name} does not match the write alias #{@ilm_write_alias}" if @ilm_write_alias != @template_name + # Overwrite the index with the write alias. + @logger.warn "Overwriting index name with rollover alias #{@ilm_write_alias}" if @index != LogStash::Outputs::ElasticSearch::CommonConfigs::DEFAULT_INDEX_NAME + @index = @ilm_write_alias verify_ilm_policy unless ilm_policy_default? end @@ -446,8 +445,6 @@ def write_alias_payload } end - - def policy_payload policy_path = ::File.expand_path(ILM_POLICY_PATH, ::File.dirname(__FILE__)) LogStash::Json.load(::IO.read(policy_path)) diff --git a/lib/logstash/outputs/elasticsearch/http_client.rb b/lib/logstash/outputs/elasticsearch/http_client.rb index 806960a6a..86d865875 100644 --- a/lib/logstash/outputs/elasticsearch/http_client.rb +++ b/lib/logstash/outputs/elasticsearch/http_client.rb @@ -362,7 +362,21 @@ def write_alias_exists?(name) # Create a new write alias def write_alias_put(alias_name, alias_definition) logger.info("Creating write alias #{alias_name}") - @pool.put(CGI::escape(alias_name), nil, LogStash::Json.dump(alias_definition)) + begin + @pool.put(CGI::escape(alias_name), nil, LogStash::Json.dump(alias_definition)) + # If the rollover alias already exists, ignore the error that comes back from Elasticsearch + # + rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e + if e.response_code == 400 && e.response_body + error_body = LogStash::Json.load(e.response_bod) + if error_body['error'] && error_body['error']['root_cause'] && + error_body['error']['root_cause'][0] && error_body['error']['root_cause'][0]['type'] == 'resource_already_exists_exception' + logger.info("Write Alias #{alias_name} already exists. Skipping") + return + end + end + raise e + end end def get_xpack_info diff --git a/lib/logstash/outputs/elasticsearch/template_manager.rb b/lib/logstash/outputs/elasticsearch/template_manager.rb index d76b8465e..115923418 100644 --- a/lib/logstash/outputs/elasticsearch/template_manager.rb +++ b/lib/logstash/outputs/elasticsearch/template_manager.rb @@ -24,7 +24,11 @@ def self.install(client, template_name, template, template_overwrite) def self.add_ilm_settings_to_template(plugin, template) # Include ilm settings in template: + plugin.logger.info("Overwriting template name and pattern as ILM is enabled.") template['template'] = "#{plugin.ilm_write_alias}-*" + if template['settings'] && (template['settings']['index.lifecycle.name'] || template['settings']['index.lifecycle.rollover_alias']) + plugin.logger.info("Overwriting index lifecycle name and rollover alias as ILM is enabled.") + end template['settings'].update({ 'index.lifecycle.name' => plugin.ilm_policy, 'index.lifecycle.rollover_alias' => plugin.ilm_write_alias}) end diff --git a/spec/es_spec_helper.rb b/spec/es_spec_helper.rb index 732a69a62..2cbbb4c94 100644 --- a/spec/es_spec_helper.rb +++ b/spec/es_spec_helper.rb @@ -3,6 +3,7 @@ require 'elasticsearch' require_relative "support/elasticsearch/api/actions/delete_ilm_policy" require_relative "support/elasticsearch/api/actions/get_alias" +require_relative "support/elasticsearch/api/actions/put_alias" require_relative "support/elasticsearch/api/actions/get_ilm_policy" require_relative "support/elasticsearch/api/actions/put_ilm_policy" @@ -96,6 +97,17 @@ def put_policy(client, policy_name, policy) client.put_ilm_policy({:name => policy_name, :body=> policy}) end + def put_alias(client, the_alias, index) + body = { + "aliases" => { + index => { + "is_write_index"=> true + } + } + } + client.put_alias({name: the_alias, body: body}) + end + def clean_ilm(client) client.get_ilm_policy.each_key {|key| client.delete_ilm_policy(name: key)} end diff --git a/spec/fixtures/template-with-policy-es7x.json b/spec/fixtures/template-with-policy-es7x.json index c8a0ab192..92de988fc 100644 --- a/spec/fixtures/template-with-policy-es7x.json +++ b/spec/fixtures/template-with-policy-es7x.json @@ -3,9 +3,7 @@ "version" : 60001, "settings" : { "index.refresh_interval" : "1s", - "number_of_shards": 1, - "index.lifecycle.name": "custom-policy", - "index.lifecycle.rollover_alias": "custom" + "number_of_shards": 1 }, "mappings" : { "_doc" : { diff --git a/spec/integration/outputs/ilm_spec.rb b/spec/integration/outputs/ilm_spec.rb index a7ab9b0af..15147192c 100644 --- a/spec/integration/outputs/ilm_spec.rb +++ b/spec/integration/outputs/ilm_spec.rb @@ -192,6 +192,7 @@ } let (:policy) { small_max_doc_policy } + let (:small_max_doc_policy) { {"policy" => { "phases"=> { @@ -416,6 +417,7 @@ before :each do put_policy(@es,ilm_policy_name, policy) end + it 'should create the write alias' do expect(@es.indices.exists_alias(index: ilm_write_alias)).to be_falsey subject.register @@ -424,6 +426,18 @@ expect(@es.get_alias(name: ilm_write_alias)).to include("#{ilm_write_alias}-#{todays_date}-000001") end + context 'when the custom write alias already exists' do + it 'should ignore the already exists error' do + expect(@es.indices.exists_alias(index: ilm_write_alias)).to be_falsey + put_alias(@es, "#{ilm_write_alias}-#{todays_date}-000001", ilm_write_alias) + expect(@es.indices.exists_alias(index: ilm_write_alias)).to be_truthy + subject.register + sleep(1) + expect(@es.get_alias(name: ilm_write_alias)).to include("#{ilm_write_alias}-#{todays_date}-000001") + end + + end + it 'should write the ILM settings into the template' do subject.register sleep(1) diff --git a/spec/support/elasticsearch/api/actions/put_alias.rb b/spec/support/elasticsearch/api/actions/put_alias.rb new file mode 100644 index 000000000..d0585934f --- /dev/null +++ b/spec/support/elasticsearch/api/actions/put_alias.rb @@ -0,0 +1,24 @@ +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. + +module Elasticsearch + module API + module Actions + + # @option arguments [String] :name The name of the alias (*Required*) + # @option arguments [Hash] :The alias definition(*Required*) + + def put_alias(arguments={}) + raise ArgumentError, "Required argument 'name' missing" unless arguments[:name] + raise ArgumentError, "Required argument 'body' missing" unless arguments[:body] + method = HTTP_PUT + path = Utils.__pathify Utils.__escape(arguments[:name]) + + params = Utils.__validate_and_extract_params arguments + body = arguments[:body] + perform_request(method, path, params, body.to_json).body + end + end + end +end diff --git a/spec/support/elasticsearch/api/actions/put_ilm_policy.rb b/spec/support/elasticsearch/api/actions/put_ilm_policy.rb index 57e8305b7..e670366b8 100644 --- a/spec/support/elasticsearch/api/actions/put_ilm_policy.rb +++ b/spec/support/elasticsearch/api/actions/put_ilm_policy.rb @@ -6,8 +6,8 @@ module Elasticsearch module API module Actions - # @option arguments [String] :name The name of the template (*Required*) - # @option arguments [Hash] :body The template definition (*Required*) + # @option arguments [String] :name The name of the policy (*Required*) + # @option arguments [Hash] :body The policy definition (*Required*) def put_ilm_policy(arguments={}) raise ArgumentError, "Required argument 'name' missing" unless arguments[:name] From 720db5551394f34cb3596bdccf9050db79d89573 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Tue, 4 Dec 2018 08:23:20 -0500 Subject: [PATCH 13/19] Rename write alias to rollover alias for consistency with beats --- lib/logstash/outputs/elasticsearch/common.rb | 23 ++++++------ .../outputs/elasticsearch/common_configs.rb | 8 ++--- .../outputs/elasticsearch/http_client.rb | 20 +++++------ .../outputs/elasticsearch/template_manager.rb | 6 ++-- spec/integration/outputs/ilm_spec.rb | 36 +++++++++---------- 5 files changed, 45 insertions(+), 48 deletions(-) diff --git a/lib/logstash/outputs/elasticsearch/common.rb b/lib/logstash/outputs/elasticsearch/common.rb index 6aa629cb1..83fb3a768 100644 --- a/lib/logstash/outputs/elasticsearch/common.rb +++ b/lib/logstash/outputs/elasticsearch/common.rb @@ -368,8 +368,9 @@ def dlq_enabled? def setup_ilm return unless ilm_enabled? - # As soon as the template is loaded, check for existence of write alias: - maybe_create_write_alias + logger.info("Using Index lifecycle management - this feature is currently in beta.") + # As soon as the template is loaded, check for existence of rollover alias: + maybe_create_rollover_alias maybe_create_ilm_policy end @@ -403,9 +404,9 @@ def verify_ilm end def verify_ilm_config - # Overwrite the index with the write alias. - @logger.warn "Overwriting index name with rollover alias #{@ilm_write_alias}" if @index != LogStash::Outputs::ElasticSearch::CommonConfigs::DEFAULT_INDEX_NAME - @index = @ilm_write_alias + # Overwrite the index with the rollover alias. + @logger.warn "Overwriting index name with rollover alias #{@ilm_rollover_alias}" if @index != LogStash::Outputs::ElasticSearch::CommonConfigs::DEFAULT_INDEX_NAME + @index = @ilm_rollover_alias verify_ilm_policy unless ilm_policy_default? end @@ -427,18 +428,18 @@ def maybe_create_ilm_policy end end - def maybe_create_write_alias - client.write_alias_put(write_alias_target, write_alias_payload) unless client.write_alias_exists?(ilm_write_alias) + def maybe_create_rollover_alias + client.rollover_alias_put(rollover_alias_target, rollover_alias_payload) unless client.rollover_alias_exists?(ilm_rollover_alias) end - def write_alias_target - "<#{ilm_write_alias}-#{ilm_pattern}>" + def rollover_alias_target + "<#{ilm_rollover_alias}-#{ilm_pattern}>" end - def write_alias_payload + def rollover_alias_payload { "aliases" => { - ilm_write_alias =>{ + ilm_rollover_alias =>{ "is_write_index" => true } } diff --git a/lib/logstash/outputs/elasticsearch/common_configs.rb b/lib/logstash/outputs/elasticsearch/common_configs.rb index 0e4bbce1b..cb0bc0d9b 100644 --- a/lib/logstash/outputs/elasticsearch/common_configs.rb +++ b/lib/logstash/outputs/elasticsearch/common_configs.rb @@ -141,21 +141,21 @@ def self.included(mod) # ----- - # ILM configurations + # ILM configurations (beta) # ----- # Flag for enabling Index Lifecycle Management integration. mod.config :ilm_enabled, :validate => :boolean, :default => false - # Write alias used for indexing data. If write alias doesn't exist, Logstash will create it and map it to the relevant index - mod.config :ilm_write_alias, :validate => :string, :default => 'logstash' + # Rollover alias used for indexing data. If rollover alias doesn't exist, Logstash will create it and map it to the relevant index + mod.config :ilm_rollover_alias, :validate => :string, :default => 'logstash' # appends “{now/d}-000001” by default for new index creation, subsequent rollover indices will increment based on this pattern i.e. “000002” + # {now/d} is date math, and will insert the appropriate value automatically. mod.config :ilm_pattern, :validate => :string, :default => '{now/d}-000001' # ILM policy to use, if undefined the default policy will be used. mod.config :ilm_policy, :validate => :string, :default => 'logstash-policy' - end end end end end diff --git a/lib/logstash/outputs/elasticsearch/http_client.rb b/lib/logstash/outputs/elasticsearch/http_client.rb index 86d865875..9335fbb66 100644 --- a/lib/logstash/outputs/elasticsearch/http_client.rb +++ b/lib/logstash/outputs/elasticsearch/http_client.rb @@ -354,28 +354,24 @@ def template_put(name, template) # ILM methods - # check whether write alias already exists - def write_alias_exists?(name) + # check whether rollover alias already exists + def rollover_alias_exists?(name) exists?(name) end - # Create a new write alias - def write_alias_put(alias_name, alias_definition) - logger.info("Creating write alias #{alias_name}") + # Create a new rollover alias + def rollover_alias_put(alias_name, alias_definition) + logger.info("Creating rollover alias #{alias_name}") begin @pool.put(CGI::escape(alias_name), nil, LogStash::Json.dump(alias_definition)) # If the rollover alias already exists, ignore the error that comes back from Elasticsearch # rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e - if e.response_code == 400 && e.response_body - error_body = LogStash::Json.load(e.response_bod) - if error_body['error'] && error_body['error']['root_cause'] && - error_body['error']['root_cause'][0] && error_body['error']['root_cause'][0]['type'] == 'resource_already_exists_exception' - logger.info("Write Alias #{alias_name} already exists. Skipping") + if e.response_code == 400 + logger.info("Rollover Alias #{alias_name} already exists. Skipping") return - end end - raise e + raise e end end diff --git a/lib/logstash/outputs/elasticsearch/template_manager.rb b/lib/logstash/outputs/elasticsearch/template_manager.rb index 115923418..a4f1ca842 100644 --- a/lib/logstash/outputs/elasticsearch/template_manager.rb +++ b/lib/logstash/outputs/elasticsearch/template_manager.rb @@ -25,16 +25,16 @@ def self.install(client, template_name, template, template_overwrite) def self.add_ilm_settings_to_template(plugin, template) # Include ilm settings in template: plugin.logger.info("Overwriting template name and pattern as ILM is enabled.") - template['template'] = "#{plugin.ilm_write_alias}-*" + template['template'] = "#{plugin.ilm_rollover_alias}-*" if template['settings'] && (template['settings']['index.lifecycle.name'] || template['settings']['index.lifecycle.rollover_alias']) plugin.logger.info("Overwriting index lifecycle name and rollover alias as ILM is enabled.") end - template['settings'].update({ 'index.lifecycle.name' => plugin.ilm_policy, 'index.lifecycle.rollover_alias' => plugin.ilm_write_alias}) + template['settings'].update({ 'index.lifecycle.name' => plugin.ilm_policy, 'index.lifecycle.rollover_alias' => plugin.ilm_rollover_alias}) end # TODO: Verify this is ok def self.template_name(plugin) - plugin.ilm_enabled? ? plugin.ilm_write_alias : plugin.template_name + plugin.ilm_enabled? ? plugin.ilm_rollover_alias : plugin.template_name end def self.default_template_path(es_major_version) diff --git a/spec/integration/outputs/ilm_spec.rb b/spec/integration/outputs/ilm_spec.rb index 15147192c..29b5c2bb7 100644 --- a/spec/integration/outputs/ilm_spec.rb +++ b/spec/integration/outputs/ilm_spec.rb @@ -258,7 +258,7 @@ context 'when using the default policy' do context 'with a custom pattern' do let (:settings) { super.merge("ilm_pattern" => "000001")} - it 'should create a write alias' do + it 'should create a rollover alias' do expect(@es.indices.exists_alias(index: "logstash")).to be_falsey subject.register sleep(1) @@ -275,7 +275,7 @@ expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.not_to raise_error end - it 'should create the default write alias' do + it 'should create the default rollover alias' do expect(@es.indices.exists_alias(index: "logstash")).to be_falsey subject.register sleep(1) @@ -376,7 +376,7 @@ context 'with the default template' do let(:expected_index) { "logstash" } - it 'should create the write alias' do + it 'should create the rollover alias' do expect(@es.indices.exists_alias(index: expected_index)).to be_falsey subject.register sleep(1) @@ -396,13 +396,13 @@ end context 'with a custom template' do - let (:ilm_write_alias) { "custom" } - let (:index) { ilm_write_alias } + let (:ilm_rollover_alias) { "custom" } + let (:index) { ilm_rollover_alias } let(:expected_index) { index } let (:settings) { super.merge("ilm_policy" => ilm_policy_name, "template" => template, "template_name" => template_name, - "ilm_write_alias" => ilm_write_alias)} + "ilm_rollover_alias" => ilm_rollover_alias)} let (:template_name) { "custom" } if ESHelper.es_version_satisfies?(">= 7.0") @@ -418,22 +418,22 @@ put_policy(@es,ilm_policy_name, policy) end - it 'should create the write alias' do - expect(@es.indices.exists_alias(index: ilm_write_alias)).to be_falsey + it 'should create the rollover alias' do + expect(@es.indices.exists_alias(index: ilm_rollover_alias)).to be_falsey subject.register sleep(1) - expect(@es.indices.exists_alias(index: ilm_write_alias)).to be_truthy - expect(@es.get_alias(name: ilm_write_alias)).to include("#{ilm_write_alias}-#{todays_date}-000001") + expect(@es.indices.exists_alias(index: ilm_rollover_alias)).to be_truthy + expect(@es.get_alias(name: ilm_rollover_alias)).to include("#{ilm_rollover_alias}-#{todays_date}-000001") end - context 'when the custom write alias already exists' do + context 'when the custom rollover alias already exists' do it 'should ignore the already exists error' do - expect(@es.indices.exists_alias(index: ilm_write_alias)).to be_falsey - put_alias(@es, "#{ilm_write_alias}-#{todays_date}-000001", ilm_write_alias) - expect(@es.indices.exists_alias(index: ilm_write_alias)).to be_truthy + expect(@es.indices.exists_alias(index: ilm_rollover_alias)).to be_falsey + put_alias(@es, "#{ilm_rollover_alias}-#{todays_date}-000001", ilm_rollover_alias) + expect(@es.indices.exists_alias(index: ilm_rollover_alias)).to be_truthy subject.register sleep(1) - expect(@es.get_alias(name: ilm_write_alias)).to include("#{ilm_write_alias}-#{todays_date}-000001") + expect(@es.get_alias(name: ilm_rollover_alias)).to include("#{ilm_rollover_alias}-#{todays_date}-000001") end end @@ -441,9 +441,9 @@ it 'should write the ILM settings into the template' do subject.register sleep(1) - expect(@es.indices.get_template(name: template_name)[template_name]["index_patterns"]).to eq(["#{ilm_write_alias}-*"]) + expect(@es.indices.get_template(name: template_name)[template_name]["index_patterns"]).to eq(["#{ilm_rollover_alias}-*"]) expect(@es.indices.get_template(name: template_name)[template_name]["settings"]['index']['lifecycle']['name']).to eq(ilm_policy_name) - expect(@es.indices.get_template(name: template_name)[template_name]["settings"]['index']['lifecycle']['rollover_alias']).to eq(ilm_write_alias) + expect(@es.indices.get_template(name: template_name)[template_name]["settings"]['index']['lifecycle']['rollover_alias']).to eq(ilm_rollover_alias) end it_behaves_like 'an ILM enabled Logstash' @@ -453,7 +453,7 @@ context 'with ilm disabled' do let (:ilm_enabled) { false } - it 'should not create a write alias' do + it 'should not create a rollover alias' do expect(@es.get_alias).to be_empty subject.register sleep(1) From 5531cb604d1e2d7927e6e2a101aef32cc7e582b9 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Tue, 4 Dec 2018 16:05:43 -0500 Subject: [PATCH 14/19] Replace use of 'template' with 'index_patterns' when amending templates --- lib/logstash/outputs/elasticsearch/template_manager.rb | 10 ++++++---- spec/fixtures/template-with-policy-es6x.json | 6 +++--- spec/fixtures/template-with-policy-es7x.json | 2 +- spec/integration/outputs/ilm_spec.rb | 6 ++++-- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/lib/logstash/outputs/elasticsearch/template_manager.rb b/lib/logstash/outputs/elasticsearch/template_manager.rb index a4f1ca842..e061b16a7 100644 --- a/lib/logstash/outputs/elasticsearch/template_manager.rb +++ b/lib/logstash/outputs/elasticsearch/template_manager.rb @@ -24,17 +24,19 @@ def self.install(client, template_name, template, template_overwrite) def self.add_ilm_settings_to_template(plugin, template) # Include ilm settings in template: - plugin.logger.info("Overwriting template name and pattern as ILM is enabled.") - template['template'] = "#{plugin.ilm_rollover_alias}-*" + plugin.logger.info("Overwriting index patterns, as ILM is enabled.") + # Overwrite any index patterns, and use the rollover alias. Use 'index_patterns' rather than 'template' for pattern + # definition - remove any existing definition of 'template' + template.delete('template') if template.include?('template') + template['index_patterns'] = "#{plugin.ilm_rollover_alias}-*" if template['settings'] && (template['settings']['index.lifecycle.name'] || template['settings']['index.lifecycle.rollover_alias']) plugin.logger.info("Overwriting index lifecycle name and rollover alias as ILM is enabled.") end template['settings'].update({ 'index.lifecycle.name' => plugin.ilm_policy, 'index.lifecycle.rollover_alias' => plugin.ilm_rollover_alias}) end - # TODO: Verify this is ok def self.template_name(plugin) - plugin.ilm_enabled? ? plugin.ilm_rollover_alias : plugin.template_name + (plugin.ilm_enabled? && !plugin.template_name) ? plugin.ilm_rollover_alias : plugin.template_name end def self.default_template_path(es_major_version) diff --git a/spec/fixtures/template-with-policy-es6x.json b/spec/fixtures/template-with-policy-es6x.json index 4566fa852..c6b89bec8 100644 --- a/spec/fixtures/template-with-policy-es6x.json +++ b/spec/fixtures/template-with-policy-es6x.json @@ -1,11 +1,11 @@ { - "template" : "custom-*", + "template" : "overwrite-*", "version" : 60001, "settings" : { "index.refresh_interval" : "1s", "number_of_shards": 1, - "index.lifecycle.name": "custom-policy", - "index.lifecycle.rollover_alias": "custom" + "index.lifecycle.name": "overwrite-policy", + "index.lifecycle.rollover_alias": "overwrite" }, "mappings" : { "_default_" : { diff --git a/spec/fixtures/template-with-policy-es7x.json b/spec/fixtures/template-with-policy-es7x.json index 92de988fc..175d9b877 100644 --- a/spec/fixtures/template-with-policy-es7x.json +++ b/spec/fixtures/template-with-policy-es7x.json @@ -1,5 +1,5 @@ { - "template" : "custom-*", + "index_patterns" : "overwrite-*", "version" : 60001, "settings" : { "index.refresh_interval" : "1s", diff --git a/spec/integration/outputs/ilm_spec.rb b/spec/integration/outputs/ilm_spec.rb index 29b5c2bb7..a4daafea4 100644 --- a/spec/integration/outputs/ilm_spec.rb +++ b/spec/integration/outputs/ilm_spec.rb @@ -396,14 +396,15 @@ end context 'with a custom template' do - let (:ilm_rollover_alias) { "custom" } + let (:ilm_rollover_alias) { "the_cat_in_the_hat" } let (:index) { ilm_rollover_alias } let(:expected_index) { index } + let (:template_name) { "custom" } let (:settings) { super.merge("ilm_policy" => ilm_policy_name, "template" => template, "template_name" => template_name, "ilm_rollover_alias" => ilm_rollover_alias)} - let (:template_name) { "custom" } + if ESHelper.es_version_satisfies?(">= 7.0") let (:template) { "spec/fixtures/template-with-policy-es7x.json" } @@ -441,6 +442,7 @@ it 'should write the ILM settings into the template' do subject.register sleep(1) + puts @es.indices.get_template(name: template_name)[template_name] expect(@es.indices.get_template(name: template_name)[template_name]["index_patterns"]).to eq(["#{ilm_rollover_alias}-*"]) expect(@es.indices.get_template(name: template_name)[template_name]["settings"]['index']['lifecycle']['name']).to eq(ilm_policy_name) expect(@es.indices.get_template(name: template_name)[template_name]["settings"]['index']['lifecycle']['rollover_alias']).to eq(ilm_rollover_alias) From f0d79e0108261548fec4939868dc193e751f5133 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Thu, 6 Dec 2018 12:29:53 -0500 Subject: [PATCH 15/19] General tidy up --- ci/run.sh | 1 - lib/logstash/outputs/elasticsearch/common.rb | 58 +++++++------------ .../outputs/elasticsearch/common_configs.rb | 4 +- .../elasticsearch-template-es6x.json | 3 +- .../outputs/elasticsearch/http_client.rb | 3 +- .../outputs/elasticsearch/template_manager.rb | 1 - spec/integration/outputs/ilm_spec.rb | 1 - spec/integration/outputs/parent_spec.rb | 2 +- spec/integration/outputs/routing_spec.rb | 50 ++++++++-------- 9 files changed, 52 insertions(+), 71 deletions(-) diff --git a/ci/run.sh b/ci/run.sh index 7f4d80db1..2bbfa3c45 100755 --- a/ci/run.sh +++ b/ci/run.sh @@ -179,7 +179,6 @@ else start_es bundle exec rspec -fd $extra_tag_args --tag update_tests:painless --tag update_tests:groovy --tag es_version:$es_distribution_version $spec_path ;; - 5.*) setup_es https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ES_VERSION}.tar.gz es_distribution_version=$(get_es_distribution_version) diff --git a/lib/logstash/outputs/elasticsearch/common.rb b/lib/logstash/outputs/elasticsearch/common.rb index 83fb3a768..3322e1e0a 100644 --- a/lib/logstash/outputs/elasticsearch/common.rb +++ b/lib/logstash/outputs/elasticsearch/common.rb @@ -1,16 +1,14 @@ require "logstash/outputs/elasticsearch/template_manager" -require 'logstash/environment' - module LogStash; module Outputs; class ElasticSearch; module Common - attr_reader :client, :hosts, :ilm_manager + attr_reader :client, :hosts # These codes apply to documents, not at the request level DOC_DLQ_CODES = [400, 404] DOC_SUCCESS_CODES = [200, 201] DOC_CONFLICT_CODE = 409 - DEFAULT_POLICY = "logstash-policy" + ILM_POLICY_PATH = "default-ilm-policy.json" # When you use external versioning, you are communicating that you want @@ -28,7 +26,7 @@ def register setup_hosts # properly sets @hosts build_client - install_template_after_successful_connection + setup_after_successful_connection check_action_validity @bulk_request_metrics = metric.namespace(:bulk_requests) @document_level_metrics = metric.namespace(:documents) @@ -43,7 +41,7 @@ def multi_receive(events) retrying_submit(events.map {|e| event_action_tuple(e)}) end - def install_template_after_successful_connection + def setup_after_successful_connection @template_installer ||= Thread.new do sleep_interval = @retry_initial_interval until successful_connection? || @stopping.true? @@ -52,9 +50,9 @@ def install_template_after_successful_connection sleep_interval = next_sleep_interval(sleep_interval) end if successful_connection? - verify_ilm + verify_ilm_readiness if ilm_enabled? install_template - setup_ilm + setup_ilm if ilm_enabled? end end end @@ -123,7 +121,6 @@ def maximum_seen_major_version client.maximum_seen_major_version end - def routing_field_name maximum_seen_major_version >= 6 ? :routing : :_routing end @@ -374,28 +371,27 @@ def setup_ilm maybe_create_ilm_policy end - def verify_ilm - return true unless ilm_enabled? - begin - verify_ilm_config + def verify_ilm_readiness + return unless ilm_enabled? - # For elasticsearch versions without ilm enablement, the _ilm endpoint will return a 400 bad request - the - # endpoint is interpreted as an index, and will return a bad request error as that is an illegal format for - # an index. + unless ilm_policy_default? || client.ilm_policy_exists?(ilm_policy) + raise LogStash::ConfigurationError, "The specified ILM policy #{ilm_policy} does not exist on your Elasticsearch instance" + end + + # Check the Elasticsearch instance for ILM readiness - this means that the version has to be a non-OSS release, with ILM feature + # available and enabled. + begin xpack = client.get_xpack_info features = xpack["features"] ilm = features["ilm"] unless features.nil? raise LogStash::ConfigurationError, "Index Lifecycle management is enabled in logstash, but not installed on your Elasticsearch cluster" if features.nil? || ilm.nil? raise LogStash::ConfigurationError, "Index Lifecycle management is enabled in logstash, but not available in your Elasticsearch cluster" unless ilm['available'] raise LogStash::ConfigurationError, "Index Lifecycle management is enabled in logstash, but not enabled in your Elasticsearch cluster" unless ilm['enabled'] - true rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e # Check xpack endpoint: If no xpack endpoint, then this version of Elasticsearch is not compatible if e.response_code == 404 - false raise LogStash::ConfigurationError, "Index Lifecycle management is enabled in logstash, but not installed on your Elasticsearch cluster" elsif e.response_code == 400 - false raise LogStash::ConfigurationError, "Index Lifecycle management is enabled in logstash, but not installed on your Elasticsearch cluster" else raise e @@ -403,23 +399,8 @@ def verify_ilm end end - def verify_ilm_config - # Overwrite the index with the rollover alias. - @logger.warn "Overwriting index name with rollover alias #{@ilm_rollover_alias}" if @index != LogStash::Outputs::ElasticSearch::CommonConfigs::DEFAULT_INDEX_NAME - @index = @ilm_rollover_alias - verify_ilm_policy unless ilm_policy_default? - end - - def ilm_policy_ok? - - end - def ilm_policy_default? - ilm_policy == DEFAULT_POLICY - end - - def verify_ilm_policy - raise LogStash::ConfigurationError, "The specified ILM policy does not exist" unless client.ilm_policy_exists?(ilm_policy) + ilm_policy == LogStash::Outputs::ElasticSearch::DEFAULT_POLICY end def maybe_create_ilm_policy @@ -429,6 +410,9 @@ def maybe_create_ilm_policy end def maybe_create_rollover_alias + @logger.warn "Overwriting supplied index name with rollover alias #{@ilm_rollover_alias}" if @index != LogStash::Outputs::ElasticSearch::CommonConfigs::DEFAULT_INDEX_NAME + @index = @ilm_rollover_alias + client.rollover_alias_put(rollover_alias_target, rollover_alias_payload) unless client.rollover_alias_exists?(ilm_rollover_alias) end @@ -438,9 +422,9 @@ def rollover_alias_target def rollover_alias_payload { - "aliases" => { + 'aliases' => { ilm_rollover_alias =>{ - "is_write_index" => true + 'is_write_index' => true } } } diff --git a/lib/logstash/outputs/elasticsearch/common_configs.rb b/lib/logstash/outputs/elasticsearch/common_configs.rb index cb0bc0d9b..ddd067884 100644 --- a/lib/logstash/outputs/elasticsearch/common_configs.rb +++ b/lib/logstash/outputs/elasticsearch/common_configs.rb @@ -4,6 +4,8 @@ module LogStash; module Outputs; class ElasticSearch module CommonConfigs DEFAULT_INDEX_NAME = "logstash-%{+YYYY.MM.dd}" + DEFAULT_POLICY = "logstash-policy" + def self.included(mod) # The index to write events to. This can be dynamic using the `%{foo}` syntax. # The default value will partition your indices by day so you can more easily @@ -154,7 +156,7 @@ def self.included(mod) mod.config :ilm_pattern, :validate => :string, :default => '{now/d}-000001' # ILM policy to use, if undefined the default policy will be used. - mod.config :ilm_policy, :validate => :string, :default => 'logstash-policy' + mod.config :ilm_policy, :validate => :string, :default => DEFAULT_POLICY end end diff --git a/lib/logstash/outputs/elasticsearch/elasticsearch-template-es6x.json b/lib/logstash/outputs/elasticsearch/elasticsearch-template-es6x.json index 9cf70ceef..2227906eb 100644 --- a/lib/logstash/outputs/elasticsearch/elasticsearch-template-es6x.json +++ b/lib/logstash/outputs/elasticsearch/elasticsearch-template-es6x.json @@ -2,8 +2,7 @@ "template" : "logstash-*", "version" : 60001, "settings" : { - "index.refresh_interval" : "5s", - "number_of_shards": 1 + "index.refresh_interval" : "5s" }, "mappings" : { "_default_" : { diff --git a/lib/logstash/outputs/elasticsearch/http_client.rb b/lib/logstash/outputs/elasticsearch/http_client.rb index 9335fbb66..32a37e82a 100644 --- a/lib/logstash/outputs/elasticsearch/http_client.rb +++ b/lib/logstash/outputs/elasticsearch/http_client.rb @@ -364,8 +364,7 @@ def rollover_alias_put(alias_name, alias_definition) logger.info("Creating rollover alias #{alias_name}") begin @pool.put(CGI::escape(alias_name), nil, LogStash::Json.dump(alias_definition)) - # If the rollover alias already exists, ignore the error that comes back from Elasticsearch - # + # If the rollover alias already exists, ignore the error that comes back from Elasticsearch rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e if e.response_code == 400 logger.info("Rollover Alias #{alias_name} already exists. Skipping") diff --git a/lib/logstash/outputs/elasticsearch/template_manager.rb b/lib/logstash/outputs/elasticsearch/template_manager.rb index e061b16a7..521424a01 100644 --- a/lib/logstash/outputs/elasticsearch/template_manager.rb +++ b/lib/logstash/outputs/elasticsearch/template_manager.rb @@ -23,7 +23,6 @@ def self.install(client, template_name, template, template_overwrite) end def self.add_ilm_settings_to_template(plugin, template) - # Include ilm settings in template: plugin.logger.info("Overwriting index patterns, as ILM is enabled.") # Overwrite any index patterns, and use the rollover alias. Use 'index_patterns' rather than 'template' for pattern # definition - remove any existing definition of 'template' diff --git a/spec/integration/outputs/ilm_spec.rb b/spec/integration/outputs/ilm_spec.rb index a4daafea4..531d9361e 100644 --- a/spec/integration/outputs/ilm_spec.rb +++ b/spec/integration/outputs/ilm_spec.rb @@ -442,7 +442,6 @@ it 'should write the ILM settings into the template' do subject.register sleep(1) - puts @es.indices.get_template(name: template_name)[template_name] expect(@es.indices.get_template(name: template_name)[template_name]["index_patterns"]).to eq(["#{ilm_rollover_alias}-*"]) expect(@es.indices.get_template(name: template_name)[template_name]["settings"]['index']['lifecycle']['name']).to eq(ilm_policy_name) expect(@es.indices.get_template(name: template_name)[template_name]["settings"]['index']['lifecycle']['rollover_alias']).to eq(ilm_rollover_alias) diff --git a/spec/integration/outputs/parent_spec.rb b/spec/integration/outputs/parent_spec.rb index c838b0e71..117ba1b8e 100644 --- a/spec/integration/outputs/parent_spec.rb +++ b/spec/integration/outputs/parent_spec.rb @@ -2,7 +2,7 @@ require "logstash/outputs/elasticsearch" if ESHelper.es_version_satisfies?("<= 5.x") - context "when using elasticsearch 5.x and before", :integration => "changethis" do + context "when using elasticsearch 5.x and before", :integration => true do shared_examples "a type based parent indexer" do let(:index) { 10.times.collect { rand(10).to_s }.join("") } let(:type) { 10.times.collect { rand(10).to_s }.join("") } diff --git a/spec/integration/outputs/routing_spec.rb b/spec/integration/outputs/routing_spec.rb index a09f91cb8..d4e23cba6 100644 --- a/spec/integration/outputs/routing_spec.rb +++ b/spec/integration/outputs/routing_spec.rb @@ -34,28 +34,28 @@ end end -# describe "(http protocol) index events with static routing", :integration => true do -# it_behaves_like 'a routing indexer' do -# let(:routing) { "test" } -# let(:config) { -# { -# "hosts" => get_host_port, -# "index" => index, -# "routing" => routing -# } -# } -# end -# end -# -# describe "(http_protocol) index events with fieldref in routing value", :integration => true do -# it_behaves_like 'a routing indexer' do -# let(:routing) { "test" } -# let(:config) { -# { -# "hosts" => get_host_port, -# "index" => index, -# "routing" => "%{message}" -# } -# } -# end -# end +describe "(http protocol) index events with static routing", :integration => true do + it_behaves_like 'a routing indexer' do + let(:routing) { "test" } + let(:config) { + { + "hosts" => get_host_port, + "index" => index, + "routing" => routing + } + } + end +end + +describe "(http_protocol) index events with fieldref in routing value", :integration => true do + it_behaves_like 'a routing indexer' do + let(:routing) { "test" } + let(:config) { + { + "hosts" => get_host_port, + "index" => index, + "routing" => "%{message}" + } + } + end +end From 6e1928787293fa24fd2817c8a3bc9dd56546f7eb Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Tue, 11 Dec 2018 12:29:04 -0500 Subject: [PATCH 16/19] Move ILM functions out of common.rb Move into their own module to be mixed in to elasticsearch.rb --- lib/logstash/outputs/elasticsearch.rb | 4 + lib/logstash/outputs/elasticsearch/common.rb | 79 ------------------ lib/logstash/outputs/elasticsearch/ilm.rb | 84 ++++++++++++++++++++ 3 files changed, 88 insertions(+), 79 deletions(-) create mode 100644 lib/logstash/outputs/elasticsearch/ilm.rb diff --git a/lib/logstash/outputs/elasticsearch.rb b/lib/logstash/outputs/elasticsearch.rb index 69e5b1d66..97ee24024 100644 --- a/lib/logstash/outputs/elasticsearch.rb +++ b/lib/logstash/outputs/elasticsearch.rb @@ -90,6 +90,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base require "logstash/outputs/elasticsearch/http_client_builder" require "logstash/outputs/elasticsearch/common_configs" require "logstash/outputs/elasticsearch/common" + require "logstash/outputs/elasticsearch/ilm" # Protocol agnostic (i.e. non-http, non-java specific) configs go here include(LogStash::Outputs::ElasticSearch::CommonConfigs) @@ -97,6 +98,9 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base # Protocol agnostic methods include(LogStash::Outputs::ElasticSearch::Common) + # Methods for ILM support + include(LogStash::Outputs::ElasticSearch::Ilm) + config_name "elasticsearch" # The Elasticsearch action to perform. Valid actions are: diff --git a/lib/logstash/outputs/elasticsearch/common.rb b/lib/logstash/outputs/elasticsearch/common.rb index 3322e1e0a..294e46495 100644 --- a/lib/logstash/outputs/elasticsearch/common.rb +++ b/lib/logstash/outputs/elasticsearch/common.rb @@ -9,8 +9,6 @@ module Common DOC_SUCCESS_CODES = [200, 201] DOC_CONFLICT_CODE = 409 - ILM_POLICY_PATH = "default-ilm-policy.json" - # When you use external versioning, you are communicating that you want # to ignore conflicts. More obviously, since an external version is a # constant part of the incoming document, we should not retry, as retrying @@ -19,7 +17,6 @@ module Common def register @template_installed = Concurrent::AtomicBoolean.new(false) - @ilm_verified = Concurrent::AtomicBoolean.new(false) @stopping = Concurrent::AtomicBoolean.new(false) # To support BWC, we check if DLQ exists in core (< 5.4). If it doesn't, we use nil to resort to previous behavior. @dlq_writer = dlq_enabled? ? execution_context.dlq_writer : nil @@ -352,87 +349,11 @@ def safe_bulk(actions) end end - def ilm_enabled? - @ilm_enabled - end - def dlq_enabled? # TODO there should be a better way to query if DLQ is enabled # See more in: https://github.com/elastic/logstash/issues/8064 respond_to?(:execution_context) && execution_context.respond_to?(:dlq_writer) && !execution_context.dlq_writer.inner_writer.is_a?(::LogStash::Util::DummyDeadLetterQueueWriter) end - - def setup_ilm - return unless ilm_enabled? - logger.info("Using Index lifecycle management - this feature is currently in beta.") - # As soon as the template is loaded, check for existence of rollover alias: - maybe_create_rollover_alias - maybe_create_ilm_policy - end - - def verify_ilm_readiness - return unless ilm_enabled? - - unless ilm_policy_default? || client.ilm_policy_exists?(ilm_policy) - raise LogStash::ConfigurationError, "The specified ILM policy #{ilm_policy} does not exist on your Elasticsearch instance" - end - - # Check the Elasticsearch instance for ILM readiness - this means that the version has to be a non-OSS release, with ILM feature - # available and enabled. - begin - xpack = client.get_xpack_info - features = xpack["features"] - ilm = features["ilm"] unless features.nil? - raise LogStash::ConfigurationError, "Index Lifecycle management is enabled in logstash, but not installed on your Elasticsearch cluster" if features.nil? || ilm.nil? - raise LogStash::ConfigurationError, "Index Lifecycle management is enabled in logstash, but not available in your Elasticsearch cluster" unless ilm['available'] - raise LogStash::ConfigurationError, "Index Lifecycle management is enabled in logstash, but not enabled in your Elasticsearch cluster" unless ilm['enabled'] - rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e - # Check xpack endpoint: If no xpack endpoint, then this version of Elasticsearch is not compatible - if e.response_code == 404 - raise LogStash::ConfigurationError, "Index Lifecycle management is enabled in logstash, but not installed on your Elasticsearch cluster" - elsif e.response_code == 400 - raise LogStash::ConfigurationError, "Index Lifecycle management is enabled in logstash, but not installed on your Elasticsearch cluster" - else - raise e - end - end - end - - def ilm_policy_default? - ilm_policy == LogStash::Outputs::ElasticSearch::DEFAULT_POLICY - end - - def maybe_create_ilm_policy - if ilm_policy_default? && !client.ilm_policy_exists?(ilm_policy) - client.ilm_policy_put(ilm_policy, policy_payload) - end - end - - def maybe_create_rollover_alias - @logger.warn "Overwriting supplied index name with rollover alias #{@ilm_rollover_alias}" if @index != LogStash::Outputs::ElasticSearch::CommonConfigs::DEFAULT_INDEX_NAME - @index = @ilm_rollover_alias - - client.rollover_alias_put(rollover_alias_target, rollover_alias_payload) unless client.rollover_alias_exists?(ilm_rollover_alias) - end - - def rollover_alias_target - "<#{ilm_rollover_alias}-#{ilm_pattern}>" - end - - def rollover_alias_payload - { - 'aliases' => { - ilm_rollover_alias =>{ - 'is_write_index' => true - } - } - } - end - - def policy_payload - policy_path = ::File.expand_path(ILM_POLICY_PATH, ::File.dirname(__FILE__)) - LogStash::Json.load(::IO.read(policy_path)) - end end end end end diff --git a/lib/logstash/outputs/elasticsearch/ilm.rb b/lib/logstash/outputs/elasticsearch/ilm.rb new file mode 100644 index 000000000..80876265b --- /dev/null +++ b/lib/logstash/outputs/elasticsearch/ilm.rb @@ -0,0 +1,84 @@ +module LogStash; module Outputs; class ElasticSearch + module Ilm + + ILM_POLICY_PATH = "default-ilm-policy.json" + + def setup_ilm + return unless ilm_enabled? + @logger.info("Using Index lifecycle management - this feature is currently in beta.") + @logger.warn "Overwriting supplied index name with rollover alias #{@ilm_rollover_alias}" if @index != LogStash::Outputs::ElasticSearch::CommonConfigs::DEFAULT_INDEX_NAME + @index = ilm_rollover_alias + + maybe_create_rollover_alias + maybe_create_ilm_policy + end + + def ilm_enabled? + @ilm_enabled + end + + def verify_ilm_readiness + return unless ilm_enabled? + + # Check the Elasticsearch instance for ILM readiness - this means that the version has to be a non-OSS release, with ILM feature + # available and enabled. + begin + xpack = client.get_xpack_info + features = xpack["features"] + ilm = features.nil? ? nil : features["ilm"] + raise LogStash::ConfigurationError, "Index Lifecycle management is enabled in logstash, but not installed on your Elasticsearch cluster" if features.nil? || ilm.nil? + raise LogStash::ConfigurationError, "Index Lifecycle management is enabled in logstash, but not available in your Elasticsearch cluster" unless ilm['available'] + raise LogStash::ConfigurationError, "Index Lifecycle management is enabled in logstash, but not enabled in your Elasticsearch cluster" unless ilm['enabled'] + + unless ilm_policy_default? || client.ilm_policy_exists?(ilm_policy) + raise LogStash::ConfigurationError, "The specified ILM policy #{ilm_policy} does not exist on your Elasticsearch instance" + end + + rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e + # Check xpack endpoint: If no xpack endpoint, then this version of Elasticsearch is not compatible + if e.response_code == 404 + raise LogStash::ConfigurationError, "Index Lifecycle management is enabled in logstash, but not installed on your Elasticsearch cluster" + elsif e.response_code == 400 + raise LogStash::ConfigurationError, "Index Lifecycle management is enabled in logstash, but not installed on your Elasticsearch cluster" + else + raise e + end + end + end + + private + + def ilm_policy_default? + ilm_policy == LogStash::Outputs::ElasticSearch::DEFAULT_POLICY + end + + def maybe_create_ilm_policy + if ilm_policy_default? && !client.ilm_policy_exists?(ilm_policy) + client.ilm_policy_put(ilm_policy, policy_payload) + end + end + + def maybe_create_rollover_alias + client.rollover_alias_put(rollover_alias_target, rollover_alias_payload) unless client.rollover_alias_exists?(ilm_rollover_alias) + end + + def rollover_alias_target + "<#{ilm_rollover_alias}-#{ilm_pattern}>" + end + + def rollover_alias_payload + { + 'aliases' => { + ilm_rollover_alias =>{ + 'is_write_index' => true + } + } + } + end + + def policy_payload + policy_path = ::File.expand_path(ILM_POLICY_PATH, ::File.dirname(__FILE__)) + LogStash::Json.load(::IO.read(policy_path)) + end + end + end end end \ No newline at end of file From affba99540a788844d261896b9889c65df41859d Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Wed, 12 Dec 2018 15:22:54 -0500 Subject: [PATCH 17/19] Cleanup template naming Also some test cleanup --- .../outputs/elasticsearch/template_manager.rb | 5 +- spec/integration/outputs/ilm_spec.rb | 46 ++++++++++++++----- 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/lib/logstash/outputs/elasticsearch/template_manager.rb b/lib/logstash/outputs/elasticsearch/template_manager.rb index 521424a01..66fad1ddf 100644 --- a/lib/logstash/outputs/elasticsearch/template_manager.rb +++ b/lib/logstash/outputs/elasticsearch/template_manager.rb @@ -34,8 +34,11 @@ def self.add_ilm_settings_to_template(plugin, template) template['settings'].update({ 'index.lifecycle.name' => plugin.ilm_policy, 'index.lifecycle.rollover_alias' => plugin.ilm_rollover_alias}) end + # Template name - if template_name set, use it + # if not and ILM is enabled, use the rollover alias + # else use the default value of template_name def self.template_name(plugin) - (plugin.ilm_enabled? && !plugin.template_name) ? plugin.ilm_rollover_alias : plugin.template_name + plugin.ilm_enabled? && !plugin.original_params.key?('template_name') ? plugin.ilm_rollover_alias : plugin.template_name end def self.default_template_path(es_major_version) diff --git a/spec/integration/outputs/ilm_spec.rb b/spec/integration/outputs/ilm_spec.rb index 531d9361e..15a3597a2 100644 --- a/spec/integration/outputs/ilm_spec.rb +++ b/spec/integration/outputs/ilm_spec.rb @@ -399,10 +399,8 @@ let (:ilm_rollover_alias) { "the_cat_in_the_hat" } let (:index) { ilm_rollover_alias } let(:expected_index) { index } - let (:template_name) { "custom" } let (:settings) { super.merge("ilm_policy" => ilm_policy_name, "template" => template, - "template_name" => template_name, "ilm_rollover_alias" => ilm_rollover_alias)} @@ -419,6 +417,8 @@ put_policy(@es,ilm_policy_name, policy) end + it_behaves_like 'an ILM enabled Logstash' + it 'should create the rollover alias' do expect(@es.indices.exists_alias(index: ilm_rollover_alias)).to be_falsey subject.register @@ -442,12 +442,26 @@ it 'should write the ILM settings into the template' do subject.register sleep(1) - expect(@es.indices.get_template(name: template_name)[template_name]["index_patterns"]).to eq(["#{ilm_rollover_alias}-*"]) - expect(@es.indices.get_template(name: template_name)[template_name]["settings"]['index']['lifecycle']['name']).to eq(ilm_policy_name) - expect(@es.indices.get_template(name: template_name)[template_name]["settings"]['index']['lifecycle']['rollover_alias']).to eq(ilm_rollover_alias) + expect(@es.indices.get_template(name: ilm_rollover_alias)[ilm_rollover_alias]["index_patterns"]).to eq(["#{ilm_rollover_alias}-*"]) + expect(@es.indices.get_template(name: ilm_rollover_alias)[ilm_rollover_alias]["settings"]['index']['lifecycle']['name']).to eq(ilm_policy_name) + expect(@es.indices.get_template(name: ilm_rollover_alias)[ilm_rollover_alias]["settings"]['index']['lifecycle']['rollover_alias']).to eq(ilm_rollover_alias) + end + + context 'with a different template_name' do + let (:template_name) { "custom_template_name" } + let (:settings) { super.merge('template_name' => template_name)} + + it_behaves_like 'an ILM enabled Logstash' + + it 'should write the ILM settings into the template' do + subject.register + sleep(1) + expect(@es.indices.get_template(name: template_name)[template_name]["index_patterns"]).to eq(["#{ilm_rollover_alias}-*"]) + expect(@es.indices.get_template(name: template_name)[template_name]["settings"]['index']['lifecycle']['name']).to eq(ilm_policy_name) + expect(@es.indices.get_template(name: template_name)[template_name]["settings"]['index']['lifecycle']['rollover_alias']).to eq(ilm_rollover_alias) + end end - it_behaves_like 'an ILM enabled Logstash' end end @@ -467,22 +481,19 @@ expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) end - it 'should write the ILM settings into the template' do + it 'should not write the ILM settings into the template' do subject.register sleep(1) expect(@es.indices.get_template(name: "logstash")["logstash"]["index_patterns"]).to eq(["logstash-*"]) expect(@es.indices.get_template(name: "logstash")["logstash"]["settings"]['index']['lifecycle']).to be_nil - end - context 'with an existing policy that will roll over' do let (:policy) { small_max_doc_policy } let (:ilm_policy_name) { "3_docs"} let (:settings) { super.merge("ilm_policy" => ilm_policy_name)} - it 'should index documents normally' do - + it 'should not roll over indices' do subject.register subject.multi_receive([ LogStash::Event.new("message" => "sample message here"), @@ -513,6 +524,19 @@ expect(indexes_written.values.first).to eq(6) end end + + context 'with a custom template name' do + let (:template_name) { "custom_template_name" } + let (:settings) { super.merge('template_name' => template_name)} + + it 'should not write the ILM settings into the template' do + subject.register + sleep(1) + expect(@es.indices.get_template(name: template_name)[template_name]["index_patterns"]).to eq(["logstash-*"]) + expect(@es.indices.get_template(name: template_name)[template_name]["settings"]['index']['lifecycle']).to be_nil + end + end + end end end \ No newline at end of file From a1b4e23421d983ab9f039d4879e5a59942178496 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Wed, 12 Dec 2018 16:32:20 -0500 Subject: [PATCH 18/19] Fix ILM specs against ES 7 --- spec/integration/outputs/ilm_spec.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/spec/integration/outputs/ilm_spec.rb b/spec/integration/outputs/ilm_spec.rb index 15a3597a2..fcbe794fa 100644 --- a/spec/integration/outputs/ilm_spec.rb +++ b/spec/integration/outputs/ilm_spec.rb @@ -63,7 +63,7 @@ # Wait or fail until everything's indexed. Stud::try(20.times) do r = @es.search - expect(r["hits"]["total"]).to eq(6) + expect(r).to have_hits(6) end indexes_written = @es.search['hits']['hits'].each_with_object(Hash.new(0)) do |x, res| index_written = x['_index'] @@ -113,7 +113,7 @@ # Wait or fail until everything's indexed. Stud::try(20.times) do r = @es.search - expect(r["hits"]["total"]).to eq(9) + expect(r).to have_hits(9) end indexes_written = @es.search['hits']['hits'].each_with_object(Hash.new(0)) do |x, res| index_written = x['_index'] @@ -154,7 +154,7 @@ # Wait or fail until everything's indexed. Stud::try(20.times) do r = @es.search - expect(r["hits"]["total"]).to eq(6) + expect(r).to have_hits(6) end indexes_written = @es.search['hits']['hits'].each_with_object(Hash.new(0)) do |x, res| index_written = x['_index'] @@ -306,7 +306,7 @@ # Wait or fail until everything's indexed. Stud::try(20.times) do r = @es.search - expect(r["hits"]["total"]).to eq(6) + expect(r).to have_hits(6) end indexes_written = @es.search['hits']['hits'].each_with_object(Hash.new(0)) do |x, res| index_written = x['_index'] @@ -514,7 +514,7 @@ # Wait or fail until everything's indexed. Stud::try(20.times) do r = @es.search - expect(r["hits"]["total"]).to eq(6) + expect(r).to have_hits(6) end indexes_written = @es.search['hits']['hits'].each_with_object(Hash.new(0)) do |x, res| index_written = x['_index'] From 38a353ff326662c650d0433cdeae9c6f01607f2c Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Thu, 13 Dec 2018 09:48:02 -0500 Subject: [PATCH 19/19] Remove redundant line` --- spec/es_spec_helper.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/spec/es_spec_helper.rb b/spec/es_spec_helper.rb index 2cbbb4c94..6e2ea466d 100644 --- a/spec/es_spec_helper.rb +++ b/spec/es_spec_helper.rb @@ -52,7 +52,6 @@ def self.es_version end RSpec::Matchers.define :have_hits do |expected| - es_version = RSpec.configuration.filter[:es_version] || ENV['ES_VERSION'] match do |actual| if ESHelper.es_version_satisfies?(">=7") expected == actual['hits']['total']['value']