diff --git a/.overcommit.yml b/.overcommit.yml new file mode 100644 index 0000000..6e3558a --- /dev/null +++ b/.overcommit.yml @@ -0,0 +1,13 @@ +PreCommit: + HardTabs: + enabled: true + + RuboCop: + enabled: true + command: ['bundle', 'exec', 'rubocop'] + + TrailingWhitespace: + enabled: true + +# Overcommit will use the repo's Gemfile when loading the Bundler context +gemfile: Gemfile diff --git a/.ruby-version b/.ruby-version new file mode 100644 index 0000000..197975a --- /dev/null +++ b/.ruby-version @@ -0,0 +1 @@ +jruby-9.2.0.0 diff --git a/CHANGELOG.md b/CHANGELOG.md index e238c92..e05ea6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ -# 1.1.0 - - improved test design to be more rspec3 friendly, including the usage - of random ports to avoid colisions in integration test. +# 2.0.0 +- Updates from logstash-output-statsd v2.0.5. +- Upgraded logstash-core dependency to 2.0. +- Upgraded dogstatsd-ruby dependency to 1.6. + +# 0.9.0 +- First version of logstash-output-dogstatsd, forked from v1.1.0 of + logstash-output-statsd. diff --git a/Gemfile b/Gemfile index d926697..851fabc 100644 --- a/Gemfile +++ b/Gemfile @@ -1,2 +1,2 @@ source 'https://rubygems.org' -gemspec \ No newline at end of file +gemspec diff --git a/README.md b/README.md index 4cd34d4..3a070f9 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,15 @@ # Logstash Plugin -This is a plugin for [Logstash](https://github.com/elasticsearch/logstash). +This is a plugin for [Logstash](https://github.com/elastic/logstash). It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way. ## Documentation -Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elasticsearch.org/guide/en/logstash/current/). +Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elastic.co/guide/en/logstash/current/). - For formatting code or config example, you can use the asciidoc `[source,ruby]` directive -- For more asciidoc formatting tips, see the excellent reference here https://github.com/elasticsearch/docs#asciidoc-guide +- For more asciidoc formatting tips, see the excellent reference here https://github.com/elastic/docs#asciidoc-guide ## Need Help? @@ -83,4 +83,4 @@ Programming is not a required skill. Whatever you've seen about open source and It is more important to the community that you are able to contribute. -For more information about contributing, see the [CONTRIBUTING](https://github.com/elasticsearch/logstash/blob/master/CONTRIBUTING.md) file. \ No newline at end of file +For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file. \ No newline at end of file diff --git a/lib/logstash/outputs/dogstatsd.rb b/lib/logstash/outputs/dogstatsd.rb new file mode 100644 index 0000000..dbc8ad8 --- /dev/null +++ b/lib/logstash/outputs/dogstatsd.rb @@ -0,0 +1,115 @@ +# encoding: utf-8 +require "logstash/outputs/base" +require "logstash/namespace" +require "datadog/statsd" + +# dogstatsd is a fork of the statsd protocol which aggregates statistics, such +# as counters and timers, and ships them over UDP to the dogstatsd-server +# running as part of the Datadog Agent. Dogstatsd adds support for metric tags, +# which are used to slice metrics along various dimensions. +# +# You can learn about statsd here: +# +# * https://codeascraft.com/2011/02/15/measure-anything-measure-everything/[Etsy blog post announcing statsd] +# * https://github.com/etsy/statsd[statsd on github] +# +# Typical examples of how this can be used with Logstash include counting HTTP hits +# by response code, summing the total number of bytes of traffic served, and tracking +# the 50th and 95th percentile of the processing time of requests. +# +# Example: +# [source,ruby] +# output { +# dogstatsd { +# metric_tags => ["host:%{host}","role:foo"] +# count => { +# "http.bytes" => "%{bytes}" +# } +# } +# } +class LogStash::Outputs::Dogstatsd < LogStash::Outputs::Base + ## Regex stolen from statsd code + RESERVED_CHARACTERS_REGEX = /[\:\|\@]/ + config_name "dogstatsd" + + # The hostname or IP address of the dogstatsd server. + config :host, :validate => :string, :default => "localhost" + + # The port to connect to on your dogstatsd server. + config :port, :validate => :number, :default => 8125 + + # An increment metric. Metric names as array. `%{fieldname}` substitutions are + # allowed in the metric names. + config :increment, :validate => :array, :default => [] + + # A decrement metric. Metric names as array. `%{fieldname}` substitutions are + # allowed in the metric names. + config :decrement, :validate => :array, :default => [] + + # A histogram metric, which a statsd timing but conceptually maps to any + # numeric value, not just durations. `metric_name => value` as hash. `%{fieldname}` + # substitutions are allowed in the metric names. + config :histogram, :validate => :hash, :default => {} + + # A count metric. `metric_name => count` as hash. `%{fieldname}` substitutions are + # allowed in the metric names. + config :count, :validate => :hash, :default => {} + + # A set metric. `metric_name => "string"` to append as hash. `%{fieldname}` + # substitutions are allowed in the metric names. + config :set, :validate => :hash, :default => {} + + # A gauge metric. `metric_name => gauge` as hash. `%{fieldname}` substitutions are + # allowed in the metric names. + config :gauge, :validate => :hash, :default => {} + + # The sample rate for the metric. + config :sample_rate, :validate => :number, :default => 1 + + # The tags to apply to each metric. + config :metric_tags, :validate => :array, :default => [] + + public + def register + @client = Datadog::Statsd.new(@host, @port) + end # def register + + public + def receive(event) + @logger.debug? and @logger.debug("Event: #{event}") + + metric_opts = { + :sample_rate => @sample_rate, + :tags => @metric_tags.map { |t| event.sprintf(t) } + } + + @increment.each do |metric| + @client.increment(event.sprintf(metric), metric_opts) + end + + @decrement.each do |metric| + @client.decrement(event.sprintf(metric), metric_opts) + end + + @count.each do |metric, val| + @client.count(event.sprintf(metric), event.sprintf(val), metric_opts) + end + + @histogram.each do |metric, val| + @client.histogram(event.sprintf(metric), event.sprintf(val), metric_opts) + end + + @set.each do |metric, val| + @client.set(event.sprintf(metric), event.sprintf(val), metric_opts) + end + + @gauge.each do |metric, val| + @client.gauge(event.sprintf(metric), event.sprintf(val), metric_opts) + end + end # def receive + + public + def close + @client.close + end # def close +end # class LogStash::Outputs::Statsd diff --git a/lib/logstash/outputs/statsd.rb b/lib/logstash/outputs/statsd.rb deleted file mode 100644 index 8769e3d..0000000 --- a/lib/logstash/outputs/statsd.rb +++ /dev/null @@ -1,128 +0,0 @@ -# encoding: utf-8 -require "logstash/outputs/base" -require "logstash/namespace" - -# statsd is a network daemon for aggregating statistics, such as counters and timers, -# and shipping over UDP to backend services, such as Graphite or Datadog. -# -# The most basic coverage of this plugin is that the 'namespace', 'sender', and -# 'metric' names are combined into the full metric path like so: -# -# `namespace.sender.metric` -# -# The general idea is that you send statsd count or latency data and every few -# seconds it will emit the aggregated values to the backend. Example aggregates are -# `average`, `max`, `stddev`, etc. -# -# You can learn about statsd here: -# -# * http://codeascraft.etsy.com/2011/02/15/measure-anything-measure-everything/[Etsy blog post announcing statsd] -# * https://github.com/etsy/statsd[statsd on github] -# -# A simple example usage of this is to count HTTP hits by response code; to learn -# more about that, check out the ../tutorials/metrics-from-logs[log metrics tutorial] -# -# The default final metric sent to statsd would look like this: -# -# `namespace.sender.metric` -# -# With regards to this plugin, the default namespace is "logstash", the default sender -# is the `${host}` field, and the metric name depends on what is set as the metric name -# in the `increment`, `decrement`, `timing`, `count, `set` or `gauge` variable. -# -# Example: -# [source,ruby] -# output { -# statsd { -# host => "statsd.example.org" -# count => { -# "http.bytes" => "%{bytes}" -# } -# } -# } -class LogStash::Outputs::Statsd < LogStash::Outputs::Base - ## Regex stolen from statsd code - RESERVED_CHARACTERS_REGEX = /[\:\|\@]/ - config_name "statsd" - - # The address of the statsd server. - config :host, :validate => :string, :default => "localhost" - - # The port to connect to on your statsd server. - config :port, :validate => :number, :default => 8125 - - # The statsd namespace to use for this metric. - config :namespace, :validate => :string, :default => "logstash" - - # The name of the sender. Dots will be replaced with underscores. - config :sender, :validate => :string, :default => "%{host}" - - # An increment metric. Metric names as array. - config :increment, :validate => :array, :default => [] - - # A decrement metric. Metric names as array. - config :decrement, :validate => :array, :default => [] - - # A timing metric. `metric_name => duration` as hash - config :timing, :validate => :hash, :default => {} - - # A count metric. `metric_name => count` as hash - config :count, :validate => :hash, :default => {} - - # A set metric. `metric_name => "string"` to append as hash - config :set, :validate => :hash, :default => {} - - # A gauge metric. `metric_name => gauge` as hash. - config :gauge, :validate => :hash, :default => {} - - # The sample rate for the metric. - config :sample_rate, :validate => :number, :default => 1 - - # Enable debugging. - config :debug, :validate => :boolean, :default => false, :deprecated => "This setting was never used by this plugin. It will be removed soon." - - public - def register - require "statsd" - @client = Statsd.new(@host, @port) - end # def register - - public - def receive(event) - return unless output?(event) - @client.namespace = event.sprintf(@namespace) if not @namespace.empty? - @logger.debug? and @logger.debug("Original sender: #{@sender}") - sender = event.sprintf(@sender) - @logger.debug? and @logger.debug("Munged sender: #{sender}") - @logger.debug? and @logger.debug("Event: #{event}") - @increment.each do |metric| - @client.increment(build_stat(event.sprintf(metric), sender), @sample_rate) - end - @decrement.each do |metric| - @client.decrement(build_stat(event.sprintf(metric), sender), @sample_rate) - end - @count.each do |metric, val| - @client.count(build_stat(event.sprintf(metric), sender), - event.sprintf(val), @sample_rate) - end - @timing.each do |metric, val| - @client.timing(build_stat(event.sprintf(metric), sender), - event.sprintf(val), @sample_rate) - end - @set.each do |metric, val| - @client.set(build_stat(event.sprintf(metric), sender), - event.sprintf(val), @sample_rate) - end - @gauge.each do |metric, val| - @client.gauge(build_stat(event.sprintf(metric), sender), - event.sprintf(val), @sample_rate) - end - end # def receive - - def build_stat(metric, sender=@sender) - sender = sender.gsub('::','.').gsub(RESERVED_CHARACTERS_REGEX, '_').gsub(".", "_") - metric = metric.gsub('::','.').gsub(RESERVED_CHARACTERS_REGEX, '_') - @logger.debug? and @logger.debug("Formatted value", :sender => sender, :metric => metric) - return "#{sender}.#{metric}" - end -end # class LogStash::Outputs::Statsd diff --git a/logstash-output-statsd.gemspec b/logstash-output-dogstatsd.gemspec similarity index 69% rename from logstash-output-statsd.gemspec rename to logstash-output-dogstatsd.gemspec index f0ae486..1d85338 100644 --- a/logstash-output-statsd.gemspec +++ b/logstash-output-dogstatsd.gemspec @@ -1,17 +1,17 @@ Gem::Specification.new do |s| - s.name = 'logstash-output-statsd' - s.version = '1.1.0' + s.name = 'logstash-output-dogstatsd' + s.version = '6.0.0' s.licenses = ['Apache License (2.0)'] s.summary = "Send metrics to StatsD" s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program" - s.authors = ["Elastic"] + s.authors = ["Tom Dooner"] s.email = 'info@elastic.co' s.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html" s.require_paths = ["lib"] # Files - s.files = `git ls-files`.split($\)+::Dir.glob('vendor/*') + s.files = Dir['lib/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT'] # Tests s.test_files = s.files.grep(%r{^(test|spec|features)/}) @@ -20,11 +20,11 @@ Gem::Specification.new do |s| s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" } # Gem dependencies - s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0' + s.add_runtime_dependency "logstash-core", "~> 6.0" s.add_runtime_dependency 'logstash-input-generator' - s.add_runtime_dependency 'statsd-ruby', ['1.2.0'] + s.add_runtime_dependency 'dogstatsd-ruby', '~> 4.0' s.add_development_dependency 'logstash-devutils' + s.add_development_dependency 'overcommit' end - diff --git a/spec/outputs/dogstatsd_spec.rb b/spec/outputs/dogstatsd_spec.rb new file mode 100644 index 0000000..0932099 --- /dev/null +++ b/spec/outputs/dogstatsd_spec.rb @@ -0,0 +1,66 @@ +# encoding: utf-8 +require_relative '../spec_helper' + +describe LogStash::Outputs::Dogstatsd do + let(:output) { described_class.new(config) } + + let(:config) do + { + 'host' => '127.0.0.1', + 'port' => 8125 + }.merge(metric_config) + end + let(:metric_config) { {} } + + describe 'registration and close' do + it 'registers without errors' do + output = LogStash::Plugin.lookup('output', 'dogstatsd').new + expect { output.register }.to_not raise_error + end + end + + describe '#send' do + before { output.register } + subject { output.receive(LogStash::Event.new(event)) } + + let(:event) { { 'something_count' => 10 } } + + context 'increment metrics' do + let(:metric_config) { { 'increment' => [metric_to_track] } } + let(:metric_to_track) { 'metric.name.here' } + + context 'with a plain ol metric name' do + it 'tracks' do + expect_any_instance_of(Datadog::Statsd).to receive(:send_to_socket) + .with("#{metric_to_track}:1|c") + subject + end + end + + context 'with tags' do + let(:metric_config) { super().merge('metric_tags' => ['foo:%{value}']) } + let(:event) { { 'value' => 'helloworld' } } + + it 'sprintf tags' do + expect_any_instance_of(Datadog::Statsd).to receive(:send_to_socket) + .with("#{metric_to_track}:1|c|#foo:helloworld") + subject + end + end + end + + context 'histogram metrics' do + let(:metric_to_track) { 'metric.name.here' } + let(:metric_config) { { 'histogram' => { '%{metric_name}' => '%{track_value}' } } } + let(:event) { super().merge('metric_name' => metric_to_track, 'track_value' => 123) } + + context 'with event fields in the metric name and value' do + it 'tracks' do + expect_any_instance_of(Datadog::Statsd).to receive(:send_to_socket) + .with("#{metric_to_track}:123|h") + subject + end + end + end + end +end diff --git a/spec/outputs/statsd_spec.rb b/spec/outputs/statsd_spec.rb deleted file mode 100644 index 3ea52fe..0000000 --- a/spec/outputs/statsd_spec.rb +++ /dev/null @@ -1,47 +0,0 @@ -# encoding: utf-8 -require "logstash/outputs/statsd" -require_relative "../spec_helper" - -describe LogStash::Outputs::Statsd do - - let(:host) { "localhost" } - let(:port) { @server.port } - - describe "registration and teardown" do - - it "should register without errors" do - output = LogStash::Plugin.lookup("output", "statsd").new - expect {output.register}.to_not raise_error - end - - end - - describe "#send" do - - context "count metrics" do - - let(:config) do - { "host" => host, "sender" => "spec", "port" => port, "count" => [ "foo.bar", "0.1" ] } - end - - let(:properties) do - { "metric" => "foo.bar", "count" => 10 } - end - - let(:event) { LogStash::Event.new(properties) } - - subject { LogStash::Outputs::Statsd.new(config) } - - before(:each) do - subject.register - end - - it "should receive data send to the server" do - subject.receive(event) - expect(@server.received).to include("logstash.spec.foo.bar:0.1|c") - end - - end - end - -end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 40723c8..e44cec1 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,69 +1,8 @@ -require "logstash/devutils/rspec/spec_helper" -require "socket" - -class StatsdServer - - attr_reader :received, :port - - def initialize - @sync_lock = Mutex.new - @terminated = false - @received = [] - end - - def register(port) - @port = port - @socket = UDPSocket.new - @socket.bind("127.0.0.1", port) - end - - def run(port) - register(port) - Thread.new do - while(!closed?) - metric, _ = @socket.recvfrom(100) - append(metric) - end - end - self - end - - def append(metric) - @sync_lock.synchronize do - @received << metric - end - end - - def close - @sync_lock.synchronize do - @terminated = true - end - end - - def closed? - @terminated == true - end - -end - -module StatdHelpers - - def random_port - rand(2000..10000) - end - -end +require 'logstash/devutils/rspec/spec_helper' +require 'logstash/outputs/dogstatsd' RSpec.configure do |c| - - c.include StatdHelpers - - c.before(:all) do - srand(c.seed) - @server = StatsdServer.new.run(random_port) - end - - c.after(:all) do - @server.close + c.before do + allow_any_instance_of(Datadog::Statsd).to receive(:send_to_socket) end end