Allow date math in pattern

Author: robbavey

lib/logstash/outputs/elasticsearch/common.rb

@@ -433,7 +433,7 @@ def maybe_create_write_alias
     end
 
     def write_alias_target
-      "#{ilm_write_alias}-#{ilm_pattern}"
+      "<#{ilm_write_alias}-#{ilm_pattern}>"
     end
 
     def write_alias_payload
@@ -446,6 +446,8 @@ def write_alias_payload
       }
     end
 
+
+
     def policy_payload
       policy_path = ::File.expand_path(ILM_POLICY_PATH, ::File.dirname(__FILE__))
       LogStash::Json.load(::IO.read(policy_path))

lib/logstash/outputs/elasticsearch/common_configs.rb

@@ -149,8 +149,8 @@ def self.included(mod)
       # Write alias used for indexing data. If write alias doesn't exist, Logstash will create it and map it to the relevant index
       mod.config :ilm_write_alias, :validate => :string, :default => 'logstash'
 
-      # appends “000001” by default for new index creation, subsequent rollover indices will increment based on this pattern i.e. “000002”
-      mod.config :ilm_pattern, :validate => :string, :default => '000001'
+      # appends “{now/d}-000001” by default for new index creation, subsequent rollover indices will increment based on this pattern i.e. “000002”
+      mod.config :ilm_pattern, :validate => :string, :default => '{now/d}-000001'
 
       # ILM policy to use, if undefined the default policy will be used.
       mod.config :ilm_policy, :validate => :string, :default => 'logstash-policy'

lib/logstash/outputs/elasticsearch/http_client.rb

@@ -362,7 +362,7 @@ def write_alias_exists?(name)
     # Create a new write alias
     def write_alias_put(alias_name, alias_definition)
       logger.info("Creating write alias #{alias_name}")
-      @pool.put(alias_name, nil, LogStash::Json.dump(alias_definition))
+      @pool.put(CGI::escape(alias_name), nil, LogStash::Json.dump(alias_definition))
     end
 
     def get_xpack_info

lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb

@@ -103,9 +103,9 @@ def format_url(url, path_and_query=nil)
       end
 
       request_uri.query = new_query_parts.join("&") unless new_query_parts.empty?
-      
-      request_uri.path = "#{request_uri.path}/#{parsed_path_and_query.path}".gsub(/\/{2,}/, "/")
-        
+
+      request_uri.path = "#{request_uri.path}/#{parsed_path_and_query.raw_path}".gsub(/\/{2,}/, "/")
+
       request_uri
     end
 

spec/es_spec_helper.rb

@@ -25,6 +25,10 @@ def doc_type
     end
   end
 
+  def todays_date
+    Time.now.strftime("%Y.%m.%d")
+  end
+
   def mapping_name
     if ESHelper.es_version_satisfies?(">=7")
       "_doc"

spec/integration/outputs/ilm_spec.rb

@@ -120,9 +120,9 @@
         res[index_written] += 1
       end
       expect(indexes_written.count).to eq(3)
-      expect(indexes_written["#{expected_index}-000001"]).to eq(3)
-      expect(indexes_written["#{expected_index}-000002"]).to eq(3)
-      expect(indexes_written["#{expected_index}-000003"]).to eq(3)
+      expect(indexes_written["#{expected_index}-#{todays_date}-000001"]).to eq(3)
+      expect(indexes_written["#{expected_index}-#{todays_date}-000002"]).to eq(3)
+      expect(indexes_written["#{expected_index}-#{todays_date}-000003"]).to eq(3)
     end
   end
 
@@ -161,7 +161,7 @@
         res[index_written] += 1
       end
       expect(indexes_written.count).to eq(1)
-      expect(indexes_written["#{expected_index}-000001"]).to eq(6)
+      expect(indexes_written["#{expected_index}-#{todays_date}-000001"]).to eq(6)
     end
   end
 end
@@ -255,6 +255,18 @@
 
 
       context 'when using the default policy' do
+        context 'with a custom pattern' do
+          let (:settings) { super.merge("ilm_pattern" => "000001")}
+          it 'should create a write alias' do
+            expect(@es.indices.exists_alias(index: "logstash")).to be_falsey
+            subject.register
+            sleep(1)
+            expect(@es.indices.exists_alias(index: "logstash")).to be_truthy
+            expect(@es.get_alias(name: "logstash")).to include("logstash-000001")
+          end
+        end
+
+
         it 'should install it if it is not present' do
           expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
           subject.register
@@ -263,14 +275,14 @@
         end
 
         it 'should create the default write alias' do
-
           expect(@es.indices.exists_alias(index: "logstash")).to be_falsey
           subject.register
           sleep(1)
           expect(@es.indices.exists_alias(index: "logstash")).to be_truthy
-          expect(@es.get_alias(name: "logstash")).to include("logstash-000001")
+          expect(@es.get_alias(name: "logstash")).to include("logstash-#{todays_date}-000001")
         end
 
+
         it 'should ingest into a single index' do
           subject.register
 
@@ -301,7 +313,7 @@
           end
 
           expect(indexes_written.count).to eq(1)
-          expect(indexes_written["logstash-000001"]).to eq(6)
+          expect(indexes_written["logstash-#{todays_date}-000001"]).to eq(6)
         end
       end
 
@@ -333,6 +345,33 @@
         end
       end
 
+      context 'when using a time based policy' do
+        let (:ilm_policy_name) {"new_one"}
+        let (:settings) { super.merge("ilm_policy" => ilm_policy_name)}
+        let (:policy) {{
+            "policy" => {
+                "phases"=> {
+                    "hot" => {
+                        "actions" => {
+                            "rollover" => {
+                                "max_age" => "1d"
+                            }
+                        }
+                    }
+                }
+            }}}
+
+        before do
+          expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
+          put_policy(@es,ilm_policy_name, policy)
+        end
+
+        it 'should not install the default policy if it is not used' do
+          subject.register
+          sleep(1)
+          expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
+        end
+      end
       context 'with the default template' do
         let(:expected_index) { "logstash" }
 
@@ -341,7 +380,7 @@
           subject.register
           sleep(1)
           expect(@es.indices.exists_alias(index: expected_index)).to be_truthy
-          expect(@es.get_alias(name: expected_index)).to include("#{expected_index}-000001")
+          expect(@es.get_alias(name: expected_index)).to include("#{expected_index}-#{todays_date}-000001")
         end
 
         it 'should write the ILM settings into the template' do
@@ -382,7 +421,7 @@
           subject.register
           sleep(1)
           expect(@es.indices.exists_alias(index: ilm_write_alias)).to be_truthy
-          expect(@es.get_alias(name: ilm_write_alias)).to include("#{ilm_write_alias}-000001")
+          expect(@es.get_alias(name: ilm_write_alias)).to include("#{ilm_write_alias}-#{todays_date}-000001")
         end
 
         it 'should write the ILM settings into the template' do