From df9731b560136bb03dfd42dc1b8c638c9ea5f70f Mon Sep 17 00:00:00 2001 From: Tom Dooner Date: Tue, 25 Aug 2015 20:40:07 +0000 Subject: [PATCH 01/10] Update gem metadata Plus-equals dog. --- ...tput-statsd.gemspec => logstash-output-dogstatsd.gemspec | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename logstash-output-statsd.gemspec => logstash-output-dogstatsd.gemspec (87%) diff --git a/logstash-output-statsd.gemspec b/logstash-output-dogstatsd.gemspec similarity index 87% rename from logstash-output-statsd.gemspec rename to logstash-output-dogstatsd.gemspec index f0ae486..be90dd0 100644 --- a/logstash-output-statsd.gemspec +++ b/logstash-output-dogstatsd.gemspec @@ -1,11 +1,11 @@ Gem::Specification.new do |s| - s.name = 'logstash-output-statsd' + s.name = 'logstash-output-dogstatsd' s.version = '1.1.0' s.licenses = ['Apache License (2.0)'] s.summary = "Send metrics to StatsD" s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program" - s.authors = ["Elastic"] + s.authors = ["Tom Dooner"] s.email = 'info@elastic.co' s.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html" s.require_paths = ["lib"] @@ -23,7 +23,7 @@ Gem::Specification.new do |s| s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0' s.add_runtime_dependency 'logstash-input-generator' - s.add_runtime_dependency 'statsd-ruby', ['1.2.0'] + s.add_runtime_dependency 'dogstatsd-ruby', '~> 1.5' s.add_development_dependency 'logstash-devutils' end From 6bd968cb5f0ebe4b068fdec17b9ec2b68352d018 Mon Sep 17 00:00:00 2001 From: Tom Dooner Date: Tue, 25 Aug 2015 20:44:15 +0000 Subject: [PATCH 02/10] Move moving files around for dogstatsd name --- CHANGELOG.md | 6 +++--- lib/logstash/outputs/{statsd.rb => dogstatsd.rb} | 0 spec/outputs/{statsd_spec.rb => dogstatsd_spec.rb} | 0 3 files changed, 3 insertions(+), 3 deletions(-) rename lib/logstash/outputs/{statsd.rb => dogstatsd.rb} (100%) rename spec/outputs/{statsd_spec.rb => dogstatsd_spec.rb} (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index e238c92..7705a1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,3 @@ -# 1.1.0 - - improved test design to be more rspec3 friendly, including the usage - of random ports to avoid colisions in integration test. +# 0.9.0 +- First version of logstash-output-dogstatsd, forked from v1.1.0 of + logstash-output-statsd. diff --git a/lib/logstash/outputs/statsd.rb b/lib/logstash/outputs/dogstatsd.rb similarity index 100% rename from lib/logstash/outputs/statsd.rb rename to lib/logstash/outputs/dogstatsd.rb diff --git a/spec/outputs/statsd_spec.rb b/spec/outputs/dogstatsd_spec.rb similarity index 100% rename from spec/outputs/statsd_spec.rb rename to spec/outputs/dogstatsd_spec.rb From f68e2913ca4990c93c2b5569497535f95ddb9be9 Mon Sep 17 00:00:00 2001 From: Tom Dooner Date: Thu, 27 Aug 2015 21:55:09 +0000 Subject: [PATCH 03/10] Convert statsd output to dogstatsd Dogstatsd is meant to be a drop-in replacement for Statsd, but there are some differences: 1) The "timing" metric type has been generalized into a "histogram", which is a cleaner conceptual fit for non-time values, e.g. file sizes 2) All metrics can be tagged, and Datadog will allow you to slice graphs and metric names by tag. This replaces the need to interpolate the sender hostname into the metric name. (Also, because the hostname is likely applied by the datadog agent's configuration.) This commit rewrites the output to support these features and depend on the dogstatsd-ruby gem instead of the statsd gem. --- .overcommit.yml | 13 ++++ .ruby-version | 1 + Gemfile | 2 +- lib/logstash/outputs/dogstatsd.rb | 98 +++++++++++++------------------ logstash-output-dogstatsd.gemspec | 1 + spec/outputs/dogstatsd_spec.rb | 76 +++++++++++++++--------- spec/spec_helper.rb | 70 ++-------------------- 7 files changed, 109 insertions(+), 152 deletions(-) create mode 100644 .overcommit.yml create mode 100644 .ruby-version diff --git a/.overcommit.yml b/.overcommit.yml new file mode 100644 index 0000000..6e3558a --- /dev/null +++ b/.overcommit.yml @@ -0,0 +1,13 @@ +PreCommit: + HardTabs: + enabled: true + + RuboCop: + enabled: true + command: ['bundle', 'exec', 'rubocop'] + + TrailingWhitespace: + enabled: true + +# Overcommit will use the repo's Gemfile when loading the Bundler context +gemfile: Gemfile diff --git a/.ruby-version b/.ruby-version new file mode 100644 index 0000000..ca6ace5 --- /dev/null +++ b/.ruby-version @@ -0,0 +1 @@ +jruby-1.7.20 diff --git a/Gemfile b/Gemfile index d926697..851fabc 100644 --- a/Gemfile +++ b/Gemfile @@ -1,2 +1,2 @@ source 'https://rubygems.org' -gemspec \ No newline at end of file +gemspec diff --git a/lib/logstash/outputs/dogstatsd.rb b/lib/logstash/outputs/dogstatsd.rb index 8769e3d..164ce0d 100644 --- a/lib/logstash/outputs/dogstatsd.rb +++ b/lib/logstash/outputs/dogstatsd.rb @@ -2,17 +2,10 @@ require "logstash/outputs/base" require "logstash/namespace" -# statsd is a network daemon for aggregating statistics, such as counters and timers, -# and shipping over UDP to backend services, such as Graphite or Datadog. -# -# The most basic coverage of this plugin is that the 'namespace', 'sender', and -# 'metric' names are combined into the full metric path like so: -# -# `namespace.sender.metric` -# -# The general idea is that you send statsd count or latency data and every few -# seconds it will emit the aggregated values to the backend. Example aggregates are -# `average`, `max`, `stddev`, etc. +# dogstatsd is a fork of the statsd protocol which aggregates statistics, such +# as counters and timers, and ships them over UDP to the dogstatsd-server +# running as part of the Datadog Agent. Dogstatsd adds support for metric tags, +# which are used to slice metrics along various dimensions. # # You can learn about statsd here: # @@ -22,49 +15,38 @@ # A simple example usage of this is to count HTTP hits by response code; to learn # more about that, check out the ../tutorials/metrics-from-logs[log metrics tutorial] # -# The default final metric sent to statsd would look like this: -# -# `namespace.sender.metric` -# -# With regards to this plugin, the default namespace is "logstash", the default sender -# is the `${host}` field, and the metric name depends on what is set as the metric name -# in the `increment`, `decrement`, `timing`, `count, `set` or `gauge` variable. -# # Example: # [source,ruby] # output { -# statsd { -# host => "statsd.example.org" -# count => { +# dogstatsd { +# metric_tags => { +# "host" => "%{host}" +# } +# count => { # "http.bytes" => "%{bytes}" # } # } # } -class LogStash::Outputs::Statsd < LogStash::Outputs::Base +class LogStash::Outputs::Dogstatsd < LogStash::Outputs::Base ## Regex stolen from statsd code RESERVED_CHARACTERS_REGEX = /[\:\|\@]/ - config_name "statsd" + config_name "dogstatsd" - # The address of the statsd server. + # The address of the dogstatsd server. config :host, :validate => :string, :default => "localhost" - # The port to connect to on your statsd server. + # The port to connect to on your dogstatsd server. config :port, :validate => :number, :default => 8125 - # The statsd namespace to use for this metric. - config :namespace, :validate => :string, :default => "logstash" - - # The name of the sender. Dots will be replaced with underscores. - config :sender, :validate => :string, :default => "%{host}" - # An increment metric. Metric names as array. config :increment, :validate => :array, :default => [] # A decrement metric. Metric names as array. config :decrement, :validate => :array, :default => [] - # A timing metric. `metric_name => duration` as hash - config :timing, :validate => :hash, :default => {} + # A histogram metric, which a statsd timing but conceptually maps to any + # numeric value, not just durations. `metric_name => value` as hash + config :histogram, :validate => :hash, :default => {} # A count metric. `metric_name => count` as hash config :count, :validate => :hash, :default => {} @@ -74,55 +56,55 @@ class LogStash::Outputs::Statsd < LogStash::Outputs::Base # A gauge metric. `metric_name => gauge` as hash. config :gauge, :validate => :hash, :default => {} - + # The sample rate for the metric. config :sample_rate, :validate => :number, :default => 1 - # Enable debugging. - config :debug, :validate => :boolean, :default => false, :deprecated => "This setting was never used by this plugin. It will be removed soon." + # The tags to apply to each metric. + config :metric_tags, :validate => :hash, :default => {} public def register - require "statsd" + require 'statsd' @client = Statsd.new(@host, @port) end # def register public def receive(event) return unless output?(event) - @client.namespace = event.sprintf(@namespace) if not @namespace.empty? - @logger.debug? and @logger.debug("Original sender: #{@sender}") - sender = event.sprintf(@sender) - @logger.debug? and @logger.debug("Munged sender: #{sender}") @logger.debug? and @logger.debug("Event: #{event}") + + tags = process_tags(event, @metric_tags) + metric_opts = { :sample_rate => @sample_rate, :tags => tags } + @increment.each do |metric| - @client.increment(build_stat(event.sprintf(metric), sender), @sample_rate) + @client.increment(event.sprintf(metric), metric_opts) end + @decrement.each do |metric| - @client.decrement(build_stat(event.sprintf(metric), sender), @sample_rate) + @client.decrement(event.sprintf(metric), metric_opts) end + @count.each do |metric, val| - @client.count(build_stat(event.sprintf(metric), sender), - event.sprintf(val), @sample_rate) + @client.count(event.sprintf(metric), event.sprintf(val), metric_opts) end - @timing.each do |metric, val| - @client.timing(build_stat(event.sprintf(metric), sender), - event.sprintf(val), @sample_rate) + + @histogram.each do |metric, val| + @client.histogram(event.sprintf(metric), event.sprintf(val), metric_opts) end + @set.each do |metric, val| - @client.set(build_stat(event.sprintf(metric), sender), - event.sprintf(val), @sample_rate) + @client.set(event.sprintf(metric), event.sprintf(val), metric_opts) end + @gauge.each do |metric, val| - @client.gauge(build_stat(event.sprintf(metric), sender), - event.sprintf(val), @sample_rate) + @client.gauge(event.sprintf(metric), event.sprintf(val), metric_opts) end end # def receive - def build_stat(metric, sender=@sender) - sender = sender.gsub('::','.').gsub(RESERVED_CHARACTERS_REGEX, '_').gsub(".", "_") - metric = metric.gsub('::','.').gsub(RESERVED_CHARACTERS_REGEX, '_') - @logger.debug? and @logger.debug("Formatted value", :sender => sender, :metric => metric) - return "#{sender}.#{metric}" + private + # Returns an array of tags like ["tag1:value1", "tag2:value2"] + def process_tags(event, tags) + tags.map { |k, v| event.sprintf(k) + ':' + event.sprintf(v) } end end # class LogStash::Outputs::Statsd diff --git a/logstash-output-dogstatsd.gemspec b/logstash-output-dogstatsd.gemspec index be90dd0..990fa83 100644 --- a/logstash-output-dogstatsd.gemspec +++ b/logstash-output-dogstatsd.gemspec @@ -26,5 +26,6 @@ Gem::Specification.new do |s| s.add_runtime_dependency 'dogstatsd-ruby', '~> 1.5' s.add_development_dependency 'logstash-devutils' + s.add_development_dependency 'overcommit' end diff --git a/spec/outputs/dogstatsd_spec.rb b/spec/outputs/dogstatsd_spec.rb index 3ea52fe..bbdfd76 100644 --- a/spec/outputs/dogstatsd_spec.rb +++ b/spec/outputs/dogstatsd_spec.rb @@ -1,47 +1,67 @@ # encoding: utf-8 -require "logstash/outputs/statsd" -require_relative "../spec_helper" +require 'logstash/outputs/dogstatsd' +require_relative '../spec_helper' -describe LogStash::Outputs::Statsd do +describe LogStash::Outputs::Dogstatsd do + let(:output) { described_class.new(config) } - let(:host) { "localhost" } - let(:port) { @server.port } - - describe "registration and teardown" do + let(:config) do + { + 'host' => '127.0.0.1', + 'port' => 8125 + }.merge(metric_config) + end + let(:metric_config) { {} } - it "should register without errors" do - output = LogStash::Plugin.lookup("output", "statsd").new - expect {output.register}.to_not raise_error + describe 'registration and teardown' do + it 'registers without errors' do + output = LogStash::Plugin.lookup('output', 'dogstatsd').new + expect { output.register }.to_not raise_error end - end - describe "#send" do + describe '#send' do + before { output.register } + subject { output.receive(LogStash::Event.new(event)) } - context "count metrics" do + let(:event) { { 'something_count' => 10 } } - let(:config) do - { "host" => host, "sender" => "spec", "port" => port, "count" => [ "foo.bar", "0.1" ] } - end + context 'increment metrics' do + let(:metric_config) { { 'increment' => [metric_to_track] } } + let(:metric_to_track) { 'metric.name.here' } - let(:properties) do - { "metric" => "foo.bar", "count" => 10 } + context 'with a plain ol metric name' do + it 'tracks' do + expect_any_instance_of(Statsd).to receive(:send_to_socket) + .with("#{metric_to_track}:1|c") + subject + end end - let(:event) { LogStash::Event.new(properties) } - - subject { LogStash::Outputs::Statsd.new(config) } + context 'with tags' do + let(:metric_config) { super().merge('metric_tags' => { 'foo' => '%{value}' }) } + let(:event) { { 'value' => 'helloworld' } } - before(:each) do - subject.register + it 'sprintf tags' do + expect_any_instance_of(Statsd).to receive(:send_to_socket) + .with("#{metric_to_track}:1|c|#foo:helloworld") + subject + end end + end - it "should receive data send to the server" do - subject.receive(event) - expect(@server.received).to include("logstash.spec.foo.bar:0.1|c") - end + context 'histogram metrics' do + let(:metric_to_track) { 'metric.name.here' } + let(:metric_config) { { 'histogram' => { '%{metric_name}' => '%{track_value}' } } } + let(:event) { super().merge('metric_name' => metric_to_track, 'track_value' => 123) } + context 'with tags in the metric name and value' do + it 'tracks' do + expect_any_instance_of(Statsd).to receive(:send_to_socket) + .with("#{metric_to_track}:123|h") + subject + end + end end end - end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 40723c8..8f6c808 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,69 +1,9 @@ -require "logstash/devutils/rspec/spec_helper" -require "socket" - -class StatsdServer - - attr_reader :received, :port - - def initialize - @sync_lock = Mutex.new - @terminated = false - @received = [] - end - - def register(port) - @port = port - @socket = UDPSocket.new - @socket.bind("127.0.0.1", port) - end - - def run(port) - register(port) - Thread.new do - while(!closed?) - metric, _ = @socket.recvfrom(100) - append(metric) - end - end - self - end - - def append(metric) - @sync_lock.synchronize do - @received << metric - end - end - - def close - @sync_lock.synchronize do - @terminated = true - end - end - - def closed? - @terminated == true - end - -end - -module StatdHelpers - - def random_port - rand(2000..10000) - end - -end +require 'logstash/devutils/rspec/spec_helper' +require 'socket' +require 'statsd' RSpec.configure do |c| - - c.include StatdHelpers - - c.before(:all) do - srand(c.seed) - @server = StatsdServer.new.run(random_port) - end - - c.after(:all) do - @server.close + c.before do + allow_any_instance_of(Statsd).to receive(:send_to_socket) end end From 21a07ad8bbc44fc4b449244abd89c53fc6275f9d Mon Sep 17 00:00:00 2001 From: Tom Dooner Date: Thu, 27 Aug 2015 23:22:07 +0000 Subject: [PATCH 04/10] Use gnarly namespacing hack to support two Statsd classes Since the dogstatsd-ruby gem authors were too lazy to rename their file from 'statsd.rb' as in the upstream statsd-ruby gem, projects like Logstash which are written in Ruby and could possibly use both have no way to differentiate between them. Luckily, Ruby lets you do whatever you want, but it's pretty gnarly. In this case, it requires a module_eval to properly namespace the Datadog Statsd class. --- lib/logstash/outputs/dogstatsd.rb | 13 +++++++++++-- logstash-output-dogstatsd.gemspec | 6 ++++-- spec/outputs/dogstatsd_spec.rb | 6 +++--- spec/spec_helper.rb | 5 ++--- 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/lib/logstash/outputs/dogstatsd.rb b/lib/logstash/outputs/dogstatsd.rb index 164ce0d..b717449 100644 --- a/lib/logstash/outputs/dogstatsd.rb +++ b/lib/logstash/outputs/dogstatsd.rb @@ -2,6 +2,16 @@ require "logstash/outputs/base" require "logstash/namespace" +# This is a hack to load the dogstatsd-ruby gem's statsd.rb file into a +# namespace so as to not clobber the top-level Statsd provided by the +# much-more-popular Statsd gem. The problem is that both gems have a file named +# 'statsd.rb' that you are supposed to load. Why did Datadog not name their file +# differently? Who knows! +# (see: https://github.com/DataDog/dogstatsd-ruby/pull/3) +module Datadog + module_eval(File.read(Gem.find_files('**/statsd.rb').grep(/dogstatsd/).first)) +end + # dogstatsd is a fork of the statsd protocol which aggregates statistics, such # as counters and timers, and ships them over UDP to the dogstatsd-server # running as part of the Datadog Agent. Dogstatsd adds support for metric tags, @@ -65,8 +75,7 @@ class LogStash::Outputs::Dogstatsd < LogStash::Outputs::Base public def register - require 'statsd' - @client = Statsd.new(@host, @port) + @client = Datadog::Statsd.new(@host, @port) end # def register public diff --git a/logstash-output-dogstatsd.gemspec b/logstash-output-dogstatsd.gemspec index 990fa83..167f26b 100644 --- a/logstash-output-dogstatsd.gemspec +++ b/logstash-output-dogstatsd.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-output-dogstatsd' - s.version = '1.1.0' + s.version = '1.2.0' s.licenses = ['Apache License (2.0)'] s.summary = "Send metrics to StatsD" s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program" @@ -23,7 +23,9 @@ Gem::Specification.new do |s| s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0' s.add_runtime_dependency 'logstash-input-generator' - s.add_runtime_dependency 'dogstatsd-ruby', '~> 1.5' + # This version is pinned exactly to ensure that upgrades don't break the + # gnarly `module_eval` hack in lib/logstash/outputs/dogstatsd.rb. + s.add_runtime_dependency 'dogstatsd-ruby', '= 1.5' s.add_development_dependency 'logstash-devutils' s.add_development_dependency 'overcommit' diff --git a/spec/outputs/dogstatsd_spec.rb b/spec/outputs/dogstatsd_spec.rb index bbdfd76..a5ff9bb 100644 --- a/spec/outputs/dogstatsd_spec.rb +++ b/spec/outputs/dogstatsd_spec.rb @@ -32,7 +32,7 @@ context 'with a plain ol metric name' do it 'tracks' do - expect_any_instance_of(Statsd).to receive(:send_to_socket) + expect_any_instance_of(Datadog::Statsd).to receive(:send_to_socket) .with("#{metric_to_track}:1|c") subject end @@ -43,7 +43,7 @@ let(:event) { { 'value' => 'helloworld' } } it 'sprintf tags' do - expect_any_instance_of(Statsd).to receive(:send_to_socket) + expect_any_instance_of(Datadog::Statsd).to receive(:send_to_socket) .with("#{metric_to_track}:1|c|#foo:helloworld") subject end @@ -57,7 +57,7 @@ context 'with tags in the metric name and value' do it 'tracks' do - expect_any_instance_of(Statsd).to receive(:send_to_socket) + expect_any_instance_of(Datadog::Statsd).to receive(:send_to_socket) .with("#{metric_to_track}:123|h") subject end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 8f6c808..e44cec1 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,9 +1,8 @@ require 'logstash/devutils/rspec/spec_helper' -require 'socket' -require 'statsd' +require 'logstash/outputs/dogstatsd' RSpec.configure do |c| c.before do - allow_any_instance_of(Statsd).to receive(:send_to_socket) + allow_any_instance_of(Datadog::Statsd).to receive(:send_to_socket) end end From 35bfe897626ce97ab836c79362e3facf2905a04f Mon Sep 17 00:00:00 2001 From: Tom Dooner Date: Fri, 28 Aug 2015 16:44:17 +0000 Subject: [PATCH 05/10] Convert metric_tags from Hash to Array You can specify the same tag multiple times, e.g. we specify "role:[chef role]" for every role in the node's run list. The syntax should support this. --- lib/logstash/outputs/dogstatsd.rb | 18 ++++++------------ logstash-output-dogstatsd.gemspec | 2 +- spec/outputs/dogstatsd_spec.rb | 4 ++-- 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/lib/logstash/outputs/dogstatsd.rb b/lib/logstash/outputs/dogstatsd.rb index b717449..5c6cb34 100644 --- a/lib/logstash/outputs/dogstatsd.rb +++ b/lib/logstash/outputs/dogstatsd.rb @@ -29,9 +29,7 @@ module Datadog # [source,ruby] # output { # dogstatsd { -# metric_tags => { -# "host" => "%{host}" -# } +# metric_tags => ["host:%{host}","role:foo"] # count => { # "http.bytes" => "%{bytes}" # } @@ -71,7 +69,7 @@ class LogStash::Outputs::Dogstatsd < LogStash::Outputs::Base config :sample_rate, :validate => :number, :default => 1 # The tags to apply to each metric. - config :metric_tags, :validate => :hash, :default => {} + config :metric_tags, :validate => :array, :default => [] public def register @@ -83,8 +81,10 @@ def receive(event) return unless output?(event) @logger.debug? and @logger.debug("Event: #{event}") - tags = process_tags(event, @metric_tags) - metric_opts = { :sample_rate => @sample_rate, :tags => tags } + metric_opts = { + :sample_rate => @sample_rate, + :tags => @metric_tags.map { |t| event.sprintf(t) } + } @increment.each do |metric| @client.increment(event.sprintf(metric), metric_opts) @@ -110,10 +110,4 @@ def receive(event) @client.gauge(event.sprintf(metric), event.sprintf(val), metric_opts) end end # def receive - - private - # Returns an array of tags like ["tag1:value1", "tag2:value2"] - def process_tags(event, tags) - tags.map { |k, v| event.sprintf(k) + ':' + event.sprintf(v) } - end end # class LogStash::Outputs::Statsd diff --git a/logstash-output-dogstatsd.gemspec b/logstash-output-dogstatsd.gemspec index 167f26b..c2d1454 100644 --- a/logstash-output-dogstatsd.gemspec +++ b/logstash-output-dogstatsd.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-output-dogstatsd' - s.version = '1.2.0' + s.version = '1.3.0' s.licenses = ['Apache License (2.0)'] s.summary = "Send metrics to StatsD" s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program" diff --git a/spec/outputs/dogstatsd_spec.rb b/spec/outputs/dogstatsd_spec.rb index a5ff9bb..368fb5d 100644 --- a/spec/outputs/dogstatsd_spec.rb +++ b/spec/outputs/dogstatsd_spec.rb @@ -39,7 +39,7 @@ end context 'with tags' do - let(:metric_config) { super().merge('metric_tags' => { 'foo' => '%{value}' }) } + let(:metric_config) { super().merge('metric_tags' => ['foo:%{value}']) } let(:event) { { 'value' => 'helloworld' } } it 'sprintf tags' do @@ -55,7 +55,7 @@ let(:metric_config) { { 'histogram' => { '%{metric_name}' => '%{track_value}' } } } let(:event) { super().merge('metric_name' => metric_to_track, 'track_value' => 123) } - context 'with tags in the metric name and value' do + context 'with event fields in the metric name and value' do it 'tracks' do expect_any_instance_of(Datadog::Statsd).to receive(:send_to_socket) .with("#{metric_to_track}:123|h") From b16b3dc0cadf394fed00678db977b8caa9c92f74 Mon Sep 17 00:00:00 2001 From: Jameel Al-Aziz Date: Mon, 29 Feb 2016 23:50:00 -0800 Subject: [PATCH 06/10] Upgrade to Logstash 2.0 API --- CHANGELOG.md | 5 ++++ README.md | 8 +++--- lib/logstash/outputs/dogstatsd.rb | 43 +++++++++++++++++++------------ logstash-output-dogstatsd.gemspec | 9 +++---- spec/outputs/dogstatsd_spec.rb | 2 +- 5 files changed, 41 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7705a1a..e05ea6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +# 2.0.0 +- Updates from logstash-output-statsd v2.0.5. +- Upgraded logstash-core dependency to 2.0. +- Upgraded dogstatsd-ruby dependency to 1.6. + # 0.9.0 - First version of logstash-output-dogstatsd, forked from v1.1.0 of logstash-output-statsd. diff --git a/README.md b/README.md index 4cd34d4..3a070f9 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,15 @@ # Logstash Plugin -This is a plugin for [Logstash](https://github.com/elasticsearch/logstash). +This is a plugin for [Logstash](https://github.com/elastic/logstash). It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way. ## Documentation -Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elasticsearch.org/guide/en/logstash/current/). +Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elastic.co/guide/en/logstash/current/). - For formatting code or config example, you can use the asciidoc `[source,ruby]` directive -- For more asciidoc formatting tips, see the excellent reference here https://github.com/elasticsearch/docs#asciidoc-guide +- For more asciidoc formatting tips, see the excellent reference here https://github.com/elastic/docs#asciidoc-guide ## Need Help? @@ -83,4 +83,4 @@ Programming is not a required skill. Whatever you've seen about open source and It is more important to the community that you are able to contribute. -For more information about contributing, see the [CONTRIBUTING](https://github.com/elasticsearch/logstash/blob/master/CONTRIBUTING.md) file. \ No newline at end of file +For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file. \ No newline at end of file diff --git a/lib/logstash/outputs/dogstatsd.rb b/lib/logstash/outputs/dogstatsd.rb index 5c6cb34..5f42e57 100644 --- a/lib/logstash/outputs/dogstatsd.rb +++ b/lib/logstash/outputs/dogstatsd.rb @@ -19,50 +19,57 @@ module Datadog # # You can learn about statsd here: # -# * http://codeascraft.etsy.com/2011/02/15/measure-anything-measure-everything/[Etsy blog post announcing statsd] +# * https://codeascraft.com/2011/02/15/measure-anything-measure-everything/[Etsy blog post announcing statsd] # * https://github.com/etsy/statsd[statsd on github] # -# A simple example usage of this is to count HTTP hits by response code; to learn -# more about that, check out the ../tutorials/metrics-from-logs[log metrics tutorial] +# Typical examples of how this can be used with Logstash include counting HTTP hits +# by response code, summing the total number of bytes of traffic served, and tracking +# the 50th and 95th percentile of the processing time of requests. # # Example: # [source,ruby] # output { -# dogstatsd { -# metric_tags => ["host:%{host}","role:foo"] -# count => { -# "http.bytes" => "%{bytes}" +# dogstatsd { +# metric_tags => ["host:%{host}","role:foo"] +# count => { +# "http.bytes" => "%{bytes}" +# } # } -# } # } class LogStash::Outputs::Dogstatsd < LogStash::Outputs::Base ## Regex stolen from statsd code RESERVED_CHARACTERS_REGEX = /[\:\|\@]/ config_name "dogstatsd" - # The address of the dogstatsd server. + # The hostname or IP address of the dogstatsd server. config :host, :validate => :string, :default => "localhost" # The port to connect to on your dogstatsd server. config :port, :validate => :number, :default => 8125 - # An increment metric. Metric names as array. + # An increment metric. Metric names as array. `%{fieldname}` substitutions are + # allowed in the metric names. config :increment, :validate => :array, :default => [] - # A decrement metric. Metric names as array. + # A decrement metric. Metric names as array. `%{fieldname}` substitutions are + # allowed in the metric names. config :decrement, :validate => :array, :default => [] # A histogram metric, which a statsd timing but conceptually maps to any - # numeric value, not just durations. `metric_name => value` as hash + # numeric value, not just durations. `metric_name => value` as hash. `%{fieldname}` + # substitutions are allowed in the metric names. config :histogram, :validate => :hash, :default => {} - # A count metric. `metric_name => count` as hash + # A count metric. `metric_name => count` as hash. `%{fieldname}` substitutions are + # allowed in the metric names. config :count, :validate => :hash, :default => {} - # A set metric. `metric_name => "string"` to append as hash + # A set metric. `metric_name => "string"` to append as hash. `%{fieldname}` + # substitutions are allowed in the metric names. config :set, :validate => :hash, :default => {} - # A gauge metric. `metric_name => gauge` as hash. + # A gauge metric. `metric_name => gauge` as hash. `%{fieldname}` substitutions are + # allowed in the metric names. config :gauge, :validate => :hash, :default => {} # The sample rate for the metric. @@ -78,7 +85,6 @@ def register public def receive(event) - return unless output?(event) @logger.debug? and @logger.debug("Event: #{event}") metric_opts = { @@ -110,4 +116,9 @@ def receive(event) @client.gauge(event.sprintf(metric), event.sprintf(val), metric_opts) end end # def receive + + public + def close + @client.close + end # def close end # class LogStash::Outputs::Statsd diff --git a/logstash-output-dogstatsd.gemspec b/logstash-output-dogstatsd.gemspec index c2d1454..0ba3d0f 100644 --- a/logstash-output-dogstatsd.gemspec +++ b/logstash-output-dogstatsd.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-output-dogstatsd' - s.version = '1.3.0' + s.version = '2.0.0' s.licenses = ['Apache License (2.0)'] s.summary = "Send metrics to StatsD" s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program" @@ -11,7 +11,7 @@ Gem::Specification.new do |s| s.require_paths = ["lib"] # Files - s.files = `git ls-files`.split($\)+::Dir.glob('vendor/*') + s.files = Dir['lib/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT'] # Tests s.test_files = s.files.grep(%r{^(test|spec|features)/}) @@ -20,14 +20,13 @@ Gem::Specification.new do |s| s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" } # Gem dependencies - s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0' + s.add_runtime_dependency "logstash-core", ">= 2.0.0.beta2", "< 3.0.0" s.add_runtime_dependency 'logstash-input-generator' # This version is pinned exactly to ensure that upgrades don't break the # gnarly `module_eval` hack in lib/logstash/outputs/dogstatsd.rb. - s.add_runtime_dependency 'dogstatsd-ruby', '= 1.5' + s.add_runtime_dependency 'dogstatsd-ruby', '= 1.6' s.add_development_dependency 'logstash-devutils' s.add_development_dependency 'overcommit' end - diff --git a/spec/outputs/dogstatsd_spec.rb b/spec/outputs/dogstatsd_spec.rb index 368fb5d..a67cc0e 100644 --- a/spec/outputs/dogstatsd_spec.rb +++ b/spec/outputs/dogstatsd_spec.rb @@ -13,7 +13,7 @@ end let(:metric_config) { {} } - describe 'registration and teardown' do + describe 'registration and close' do it 'registers without errors' do output = LogStash::Plugin.lookup('output', 'dogstatsd').new expect { output.register }.to_not raise_error From ed825ee4043be10721c52639fa50e1c8af2b0cb3 Mon Sep 17 00:00:00 2001 From: Robert Saveland Date: Tue, 10 Oct 2017 14:06:31 -0400 Subject: [PATCH 07/10] updated rbenv env to latest 1.7.27 jruby --- .ruby-version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ruby-version b/.ruby-version index ca6ace5..5d83b41 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -jruby-1.7.20 +jruby-1.7.27 From 185bb595684c36522b06c0f89269e4b9915ea063 Mon Sep 17 00:00:00 2001 From: Robert Saveland Date: Tue, 10 Oct 2017 14:07:55 -0400 Subject: [PATCH 08/10] changed dependencies to logstash 5 and dogstats 3, and removed the namespacing hack now that datadog properly namespaces Statsd --- lib/logstash/outputs/dogstatsd.rb | 11 +---------- logstash-output-dogstatsd.gemspec | 8 +++----- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/lib/logstash/outputs/dogstatsd.rb b/lib/logstash/outputs/dogstatsd.rb index 5f42e57..dbc8ad8 100644 --- a/lib/logstash/outputs/dogstatsd.rb +++ b/lib/logstash/outputs/dogstatsd.rb @@ -1,16 +1,7 @@ # encoding: utf-8 require "logstash/outputs/base" require "logstash/namespace" - -# This is a hack to load the dogstatsd-ruby gem's statsd.rb file into a -# namespace so as to not clobber the top-level Statsd provided by the -# much-more-popular Statsd gem. The problem is that both gems have a file named -# 'statsd.rb' that you are supposed to load. Why did Datadog not name their file -# differently? Who knows! -# (see: https://github.com/DataDog/dogstatsd-ruby/pull/3) -module Datadog - module_eval(File.read(Gem.find_files('**/statsd.rb').grep(/dogstatsd/).first)) -end +require "datadog/statsd" # dogstatsd is a fork of the statsd protocol which aggregates statistics, such # as counters and timers, and ships them over UDP to the dogstatsd-server diff --git a/logstash-output-dogstatsd.gemspec b/logstash-output-dogstatsd.gemspec index 0ba3d0f..1c368d1 100644 --- a/logstash-output-dogstatsd.gemspec +++ b/logstash-output-dogstatsd.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-output-dogstatsd' - s.version = '2.0.0' + s.version = '5.0.0' s.licenses = ['Apache License (2.0)'] s.summary = "Send metrics to StatsD" s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program" @@ -20,12 +20,10 @@ Gem::Specification.new do |s| s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" } # Gem dependencies - s.add_runtime_dependency "logstash-core", ">= 2.0.0.beta2", "< 3.0.0" + s.add_runtime_dependency "logstash-core", "~> 5.0" s.add_runtime_dependency 'logstash-input-generator' - # This version is pinned exactly to ensure that upgrades don't break the - # gnarly `module_eval` hack in lib/logstash/outputs/dogstatsd.rb. - s.add_runtime_dependency 'dogstatsd-ruby', '= 1.6' + s.add_runtime_dependency 'dogstatsd-ruby', '~> 3.0' s.add_development_dependency 'logstash-devutils' s.add_development_dependency 'overcommit' From 78399217370bce5b01bb47b5d4c78c547e7df74a Mon Sep 17 00:00:00 2001 From: Robert Saveland Date: Tue, 10 Oct 2017 14:08:23 -0400 Subject: [PATCH 09/10] removed require that is already in spec_helper --- spec/outputs/dogstatsd_spec.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/spec/outputs/dogstatsd_spec.rb b/spec/outputs/dogstatsd_spec.rb index a67cc0e..0932099 100644 --- a/spec/outputs/dogstatsd_spec.rb +++ b/spec/outputs/dogstatsd_spec.rb @@ -1,5 +1,4 @@ # encoding: utf-8 -require 'logstash/outputs/dogstatsd' require_relative '../spec_helper' describe LogStash::Outputs::Dogstatsd do From 694cf6935e4fbf7afb6e3fc495e02ec92f2819e2 Mon Sep 17 00:00:00 2001 From: Ahmad Wilson Date: Tue, 2 Oct 2018 00:13:01 +0000 Subject: [PATCH 10/10] Add Logstash 6.x vesion compatibility --- .ruby-version | 2 +- logstash-output-dogstatsd.gemspec | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.ruby-version b/.ruby-version index 5d83b41..197975a 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -jruby-1.7.27 +jruby-9.2.0.0 diff --git a/logstash-output-dogstatsd.gemspec b/logstash-output-dogstatsd.gemspec index 1c368d1..1d85338 100644 --- a/logstash-output-dogstatsd.gemspec +++ b/logstash-output-dogstatsd.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-output-dogstatsd' - s.version = '5.0.0' + s.version = '6.0.0' s.licenses = ['Apache License (2.0)'] s.summary = "Send metrics to StatsD" s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program" @@ -20,10 +20,10 @@ Gem::Specification.new do |s| s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" } # Gem dependencies - s.add_runtime_dependency "logstash-core", "~> 5.0" + s.add_runtime_dependency "logstash-core", "~> 6.0" s.add_runtime_dependency 'logstash-input-generator' - s.add_runtime_dependency 'dogstatsd-ruby', '~> 3.0' + s.add_runtime_dependency 'dogstatsd-ruby', '~> 4.0' s.add_development_dependency 'logstash-devutils' s.add_development_dependency 'overcommit'