Integration test improvements

Add more tests to check correct creation of write alias and templates
Tests for default settings

Author: robbavey

spec/es_spec_helper.rb

@@ -2,6 +2,7 @@
 require 'manticore'
 require 'elasticsearch'
 require_relative "support/elasticsearch/api/actions/delete_ilm_policy"
+require_relative "support/elasticsearch/api/actions/get_alias"
 require_relative "support/elasticsearch/api/actions/get_ilm_policy"
 require_relative "support/elasticsearch/api/actions/put_ilm_policy"
 

spec/integration/outputs/ilm_spec.rb

@@ -120,9 +120,9 @@
         res[index_written] += 1
       end
       expect(indexes_written.count).to eq(3)
-      expect(indexes_written["#{ilm_write_alias}-000001"]).to eq(3)
-      expect(indexes_written["#{ilm_write_alias}-000002"]).to eq(3)
-      expect(indexes_written["#{ilm_write_alias}-000003"]).to eq(3)
+      expect(indexes_written["#{expected_index}-000001"]).to eq(3)
+      expect(indexes_written["#{expected_index}-000002"]).to eq(3)
+      expect(indexes_written["#{expected_index}-000003"]).to eq(3)
     end
   end
 
@@ -161,7 +161,7 @@
         res[index_written] += 1
       end
       expect(indexes_written.count).to eq(1)
-      expect(indexes_written["#{ilm_write_alias}-000001"]).to eq(6)
+      expect(indexes_written["#{expected_index}-000001"]).to eq(6)
     end
   end
 end
@@ -182,18 +182,11 @@
     DEFAULT_INTERVAL = '600s'
 
     require "logstash/outputs/elasticsearch"
-    let (:ilm_write_alias) { "the_write_alias" }
-    let (:index) { ilm_write_alias }
     let (:ilm_enabled) { true }
 
     let (:settings) {
       {
-          "index" => index,
           "ilm_enabled" => ilm_enabled,
-          "ilm_write_alias" => ilm_write_alias,
-          "manage_template" => true,
-          "template_name" => ilm_write_alias,
-          "template_overwrite" => true,
           "hosts" => "#{get_host_port()}"
       }
     }
@@ -260,15 +253,57 @@
     context 'with ilm enabled' do
       let (:ilm_enabled) { true }
 
+
       context 'when using the default policy' do
         it 'should install it if it is not present' do
           expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
           subject.register
           sleep(1)
           expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.not_to raise_error
         end
-      end
 
+        it 'should create the default write alias' do
+
+          expect(@es.indices.exists_alias(index: "logstash")).to be_falsey
+          subject.register
+          sleep(1)
+          expect(@es.indices.exists_alias(index: "logstash")).to be_truthy
+          expect(@es.get_alias(name: "logstash")).to include("logstash-000001")
+        end
+
+        it 'should ingest into a single index' do
+          subject.register
+
+          subject.multi_receive([
+                                    LogStash::Event.new("message" => "sample message here"),
+                                    LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }),
+                                    LogStash::Event.new("somevalue" => 100),
+                                ])
+
+          sleep(6)
+
+          subject.multi_receive([
+                                    LogStash::Event.new("country" => "us"),
+                                    LogStash::Event.new("country" => "at"),
+                                    LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] })
+                                ])
+
+          @es.indices.refresh
+
+          # Wait or fail until everything's indexed.
+          Stud::try(20.times) do
+            r = @es.search
+            expect(r["hits"]["total"]).to eq(6)
+          end
+          indexes_written = @es.search['hits']['hits'].each_with_object(Hash.new(0)) do |x, res|
+            index_written = x['_index']
+            res[index_written] += 1
+          end
+
+          expect(indexes_written.count).to eq(1)
+          expect(indexes_written["logstash-000001"]).to eq(6)
+        end
+      end
 
       context 'when not using the default policy' do
         let (:ilm_policy_name) {"new_one"}
@@ -299,12 +334,22 @@
       end
 
       context 'with the default template' do
+        let(:expected_index) { "logstash" }
 
-        it 'should write the write alias' do
-          expect(@es.indices.exists_alias(index: ilm_write_alias)).to be_falsey
+        it 'should create the write alias' do
+          expect(@es.indices.exists_alias(index: expected_index)).to be_falsey
           subject.register
           sleep(1)
-          expect(@es.indices.exists_alias(index: ilm_write_alias)).to be_truthy
+          expect(@es.indices.exists_alias(index: expected_index)).to be_truthy
+          expect(@es.get_alias(name: expected_index)).to include("#{expected_index}-000001")
+        end
+
+        it 'should write the ILM settings into the template' do
+          subject.register
+          sleep(1)
+          expect(@es.indices.get_template(name: "logstash")["logstash"]["index_patterns"]).to eq(["logstash-*"])
+          expect(@es.indices.get_template(name: "logstash")["logstash"]["settings"]['index']['lifecycle']['name']).to eq("logstash-policy")
+          expect(@es.indices.get_template(name: "logstash")["logstash"]["settings"]['index']['lifecycle']['rollover_alias']).to eq("logstash")
         end
 
         it_behaves_like 'an ILM enabled Logstash'
@@ -313,24 +358,39 @@
       context 'with a custom template' do
         let (:ilm_write_alias) { "custom" }
         let (:index) { ilm_write_alias }
+        let(:expected_index) { index }
+        let (:settings) { super.merge("ilm_policy" => ilm_policy_name,
+                                      "template" => template,
+                                      "template_name" => template_name,
+                                      "ilm_write_alias" => ilm_write_alias)}
         let (:template_name) { "custom" }
+
         if ESHelper.es_version_satisfies?(">= 7.0")
           let (:template) { "spec/fixtures/template-with-policy-es7x.json" }
         else
           let (:template) { "spec/fixtures/template-with-policy-es6x.json" }
         end
         let (:ilm_enabled) { true }
         let (:ilm_policy_name) { "custom-policy" }
-        let (:settings) { super.merge("ilm_policy" => ilm_policy_name, "template" => template)}
+
 
         before :each do
           put_policy(@es,ilm_policy_name, policy)
         end
-        it 'should write the write alias' do
+        it 'should create the write alias' do
           expect(@es.indices.exists_alias(index: ilm_write_alias)).to be_falsey
           subject.register
           sleep(1)
           expect(@es.indices.exists_alias(index: ilm_write_alias)).to be_truthy
+          expect(@es.get_alias(name: ilm_write_alias)).to include("#{ilm_write_alias}-000001")
+        end
+
+        it 'should write the ILM settings into the template' do
+          subject.register
+          sleep(1)
+          expect(@es.indices.get_template(name: template_name)[template_name]["index_patterns"]).to eq(["#{ilm_write_alias}-*"])
+          expect(@es.indices.get_template(name: template_name)[template_name]["settings"]['index']['lifecycle']['name']).to eq(ilm_policy_name)
+          expect(@es.indices.get_template(name: template_name)[template_name]["settings"]['index']['lifecycle']['rollover_alias']).to eq(ilm_write_alias)
         end
 
         it_behaves_like 'an ILM enabled Logstash'
@@ -340,11 +400,11 @@
     context 'with ilm disabled' do
       let (:ilm_enabled) { false }
 
-      it 'should not write the write alias' do
-        expect(@es.indices.exists_alias(index: ilm_write_alias)).to be_falsey
+      it 'should not create a write alias' do
+        expect(@es.get_alias).to be_empty
         subject.register
         sleep(1)
-        expect(@es.indices.exists_alias(index: ilm_write_alias)).to be_falsey
+        expect(@es.get_alias).to be_empty
       end
 
       it 'should not install the default policy' do
@@ -353,6 +413,15 @@
         expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
       end
 
+      it 'should write the ILM settings into the template' do
+        subject.register
+        sleep(1)
+        expect(@es.indices.get_template(name: "logstash")["logstash"]["index_patterns"]).to eq(["logstash-*"])
+        expect(@es.indices.get_template(name: "logstash")["logstash"]["settings"]['index']['lifecycle']).to be_nil
+
+      end
+
+
       context 'with an existing policy that will roll over' do
         let (:policy) { small_max_doc_policy }
         let (:ilm_policy_name) { "3_docs"}
@@ -387,7 +456,7 @@
             res[index_written] += 1
           end
           expect(indexes_written.count).to eq(1)
-          expect(indexes_written["#{index}"]).to eq(6)
+          expect(indexes_written.values.first).to eq(6)
         end
       end
     end

spec/support/elasticsearch/api/actions/get_alias.rb

@@ -0,0 +1,18 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+module Elasticsearch
+  module API
+    module Actions
+
+      # Retrieve the list of index lifecycle management policies
+      def get_alias(arguments={})
+        method = HTTP_GET
+        path   = Utils.__pathify '_alias', Utils.__escape(arguments[:name])
+        params = {}
+        perform_request(method, path, params, nil).body
+      end
+    end
+  end
+end