From 2ea9ffc974363f11abd1453a5cdf818320852779 Mon Sep 17 00:00:00 2001 From: Marc Powell Date: Wed, 18 May 2016 15:42:43 -0500 Subject: [PATCH] Add date_prefix configuration option This change adds an optional new 'date_prefix' configuration option that will allow for the creation of a date-based prefix in the S3 path. This will work in combination with the existing 'prefix' option if desired. Any value for this option is expected to be a time definition suitable for passing through strftime (e.g '%Y-%m-%d/'). The value of this option will be inserted between 'prefix' and the filename generated by logstash. This is particularly useful if you have large volumes of logs that you wish to keep for long periods of time (archival purposes). For example, a 5 minute time_file setting will result in 156,960 files for an 18 month retention period for each host sending data. It is more convenient to work with that number of files, particularly in GUI based tools, if they are distributed into smaller groups of files. This will also allow for more granular loading of files into Elasticsearch. --- lib/logstash/outputs/s3.rb | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/lib/logstash/outputs/s3.rb b/lib/logstash/outputs/s3.rb index a8adc262..7de0c848 100644 --- a/lib/logstash/outputs/s3.rb +++ b/lib/logstash/outputs/s3.rb @@ -118,6 +118,10 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base # Specify a prefix to the uploaded filename, this can simulate directories on S3. Prefix does not require leading slash. config :prefix, :validate => :string, :default => '' + # Specify a date-based prefix to the uploaded filename, this can simulate directories on S3. Prefix does not require leading slash. + # The value will be parsed through strftime() + config :date_prefix, :validate => :string, :default => '' + # Specify how many workers to use to upload the files to S3 config :upload_workers_count, :validate => :number, :default => 1 @@ -180,7 +184,7 @@ def write_on_bucket(file) # find and use the bucket bucket = @s3.buckets[@bucket] - remote_filename = "#{@prefix}#{File.basename(file)}" + remote_filename = get_remote_filename(file) @logger.debug("S3: ready to write file in bucket", :remote_filename => remote_filename, :bucket => @bucket) @@ -328,6 +332,16 @@ def get_temporary_filename(page_counter = 0) end end + public + def get_remote_filename(file) + current_time = Time.now + if @date_prefix.to_s == '' + return "#{@prefix}#{File.basename(file)}" + else + return "#{@prefix}#{current_time.strftime(@date_prefix)}#{File.basename(file)}" + end + end + public def receive(event) @@ -472,7 +486,7 @@ def reset_page_counter def delete_on_bucket(filename) bucket = @s3.buckets[@bucket] - remote_filename = "#{@prefix}#{File.basename(filename)}" + remote_filename = get_remote_filename(filename) @logger.debug("S3: delete file from bucket", :remote_filename => remote_filename, :bucket => @bucket)