From 62e727a641403619100ec8b4f1380b75e16ddb20 Mon Sep 17 00:00:00 2001 From: Jason Zandona Date: Fri, 23 Mar 2018 10:46:49 -0700 Subject: [PATCH] Added support for endpoint based compatible S3 --- lib/logstash/inputs/s3.rb | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/lib/logstash/inputs/s3.rb b/lib/logstash/inputs/s3.rb index c3f46e4..d03e650 100644 --- a/lib/logstash/inputs/s3.rb +++ b/lib/logstash/inputs/s3.rb @@ -26,6 +26,8 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base include LogStash::PluginMixins::AwsConfig::V2 config_name "s3" + + default :codec, "plain" @@ -68,6 +70,15 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base # default to the current OS temporary directory in linux /tmp/logstash config :temporary_directory, :validate => :string, :default => File.join(Dir.tmpdir, "logstash") + # Specify a custom endpoint for use with non-AWS S3 implementations, e.g., + # Ceph. Provide a URL in the format http://127.0.0.1:8080/ + config :endpoint, :validate => :string + + # When false, specify the bucket in the subdomain. When true, specify the bucket in the path. + config :force_path_style, :validate => :boolean, :default => false + + + public def register require "fileutils" @@ -76,6 +87,7 @@ def register @logger.info("Registering s3 input", :bucket => @bucket, :region => @region) + s3 = get_s3object @s3bucket = s3.bucket(@bucket) @@ -386,8 +398,32 @@ def delete_file_from_bucket(object) private def get_s3object - s3 = Aws::S3::Resource.new(aws_options_hash) + s3 = Aws::S3::Resource.new(full_options) end + + + def full_options + options = Hash.new + options.merge(aws_options_hash) + .merge(endpoint_options) + end + + def endpoint_options + if @endpoint + uri = URI(@endpoint) + { + :endpoint => @endpoint, + :force_path_style => @force_path_style, + } + else + {} + end + end + + def bucket_resource + Aws::S3::Bucket.new(@bucket, full_options) + end + private module SinceDB