From a1f3b6dd150e0da2075a499176cdb8b1da34b330 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Mon, 29 Aug 2016 10:56:55 -0400 Subject: [PATCH 1/5] new concurrency model :shared --- lib/logstash/outputs/s3.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/logstash/outputs/s3.rb b/lib/logstash/outputs/s3.rb index a8a12a7e..4d751474 100644 --- a/lib/logstash/outputs/s3.rb +++ b/lib/logstash/outputs/s3.rb @@ -79,6 +79,8 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base config_name "s3" default :codec, 'line' + concurrency :shared + # S3 bucket config :bucket, :validate => :string @@ -214,8 +216,6 @@ def register # http://ruby.awsblog.com/post/Tx16QY1CI5GVBFT/Threading-with-the-AWS-SDK-for-Ruby AWS.eager_autoload!(AWS::S3) - workers_not_supported - @s3 = aws_s3_config @upload_queue = Queue.new @file_rotation_lock = Mutex.new From a8dd1890c85381d1ac20f55d94d5714493121a61 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Thu, 8 Sep 2016 10:51:54 -0400 Subject: [PATCH 2/5] Uses :singles as the concurrency model --- lib/logstash/outputs/s3.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/logstash/outputs/s3.rb b/lib/logstash/outputs/s3.rb index 4d751474..7d7f3c24 100644 --- a/lib/logstash/outputs/s3.rb +++ b/lib/logstash/outputs/s3.rb @@ -79,7 +79,7 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base config_name "s3" default :codec, 'line' - concurrency :shared + concurrency :single # S3 bucket config :bucket, :validate => :string From b5b3803602a0b18ff7aefd2e3200c276d5bd9313 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Thu, 8 Sep 2016 10:52:05 -0400 Subject: [PATCH 3/5] Update to 3.2.0 --- CHANGELOG.md | 7 +++++++ logstash-output-s3.gemspec | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a275c031..70d0d6f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,13 @@ +## 3.2.0 + - Move to the new concurrency model `:single` + - use correct license identifier #99 + - add support for `bucket_owner_full_control` in the canned ACL #87 + - dont delete plugin test files, since we only actually need the put object permission #85 + ## 3.1.2 - Fix improper shutdown of output worker threads - improve exception handling + ## 3.0.1 - Republish all the gems under jruby. diff --git a/logstash-output-s3.gemspec b/logstash-output-s3.gemspec index ba9f7e1e..3ed6eab6 100644 --- a/logstash-output-s3.gemspec +++ b/logstash-output-s3.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-output-s3' - s.version = '3.1.2' + s.version = '3.2.0' s.licenses = ['Apache-2.0'] s.summary = "This plugin was created for store the logstash's events into Amazon Simple Storage Service (Amazon S3)" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From c3fc9f2e0952cc494adb6a2a971c7b9295bdccf8 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Thu, 8 Sep 2016 11:08:03 -0400 Subject: [PATCH 4/5] Delete test files on the remote bucket We are reverting the changes we did in #85, if we dont delete the files the files can polute the bucket. So we will try to delete them and ignore if it fails. Because in reality we only need the permission to put an object for this plugin to correctly work. Fixes: #97 --- lib/logstash/outputs/s3.rb | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/logstash/outputs/s3.rb b/lib/logstash/outputs/s3.rb index 7d7f3c24..257ef357 100644 --- a/lib/logstash/outputs/s3.rb +++ b/lib/logstash/outputs/s3.rb @@ -258,6 +258,15 @@ def test_s3_write begin write_on_bucket(test_filename) + + begin + remote_filename = "#{@prefix}#{File.basename(test_filename)}" + bucket = @s3.buckets[@bucket] + bucket.objects[remote_filename].delete + rescue StandardError => e + # we actually only need `put_object`, but if we dont delete them + # we can have a lot of tests files + end ensure File.delete(test_filename) end From 808b6e0ee8b0511e918685cc368b9b332369623e Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Thu, 8 Sep 2016 11:08:27 -0400 Subject: [PATCH 5/5] fix typo in the changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70d0d6f9..093937d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ - Move to the new concurrency model `:single` - use correct license identifier #99 - add support for `bucket_owner_full_control` in the canned ACL #87 - - dont delete plugin test files, since we only actually need the put object permission #85 + - delete the test file but ignore any errors, because we actually only need to be able to write to S3. #97 ## 3.1.2 - Fix improper shutdown of output worker threads