From 29e110909a1ff1b71b637879db7a96e2c5ab2d87 Mon Sep 17 00:00:00 2001 From: Tero Saarni Date: Wed, 2 Aug 2023 19:03:18 +0300 Subject: [PATCH 1/3] Added support for RFC5424 structured data Signed-off-by: Tero Saarni --- CHANGELOG.md | 2 +- docs/index.asciidoc | 15 ++++++++++++++- lib/logstash/outputs/syslog.rb | 13 +++++++++++-- spec/outputs/syslog_spec.rb | 24 ++++++++++++++++++++++++ 4 files changed, 50 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f8167c7..872c05f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## 3.0.6 - Change codec instance comparison [#69](https://github.com/logstash-plugins/logstash-output-syslog/pull/69) + - Added support for RFC5424 structured data [#67](https://github.com/logstash-plugins/logstash-output-syslog/pull/67) ## 3.0.5 - Docs: Set the default_codec doc attribute. @@ -40,4 +41,3 @@ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895 - Dependency on logstash-core update to 2.0 - diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 11e6b7a..e824d7f 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -59,6 +59,7 @@ This plugin supports the following configuration options plus the <> |<>|No | <> |<>|No | <> |<>|No +| <> |<>|No |======================================================================= Also see <> for a list of options supported by all @@ -234,9 +235,21 @@ Verify the identity of the other end of the SSL connection against the CA. use label parsing for severity and facility levels use priority field if set to false +[id="plugins-{type}s-{plugin}-structured_data"] +===== `structured_data` + * Value type is <> + * Default value is `nil` + +RFC5424 structured data is a string of one or more structured data elements, including brackets. +The elements need to be formatted according to link:https://datatracker.ietf.org/doc/html/rfc5424#section-6.3[RFC5424 section 6.3], for example: + +[source,ruby] + `[exampleSDID@32473 iut="3" eventSource="Application" eventID="1011"][examplePriority@32473 class="high"]` + +The new value can include `%{foo}` strings to help you build a new value from other parts of the event. [id="plugins-{type}s-{plugin}-common-options"] include::{include_path}/{type}.asciidoc[] -:default_codec!: \ No newline at end of file +:default_codec!: diff --git a/lib/logstash/outputs/syslog.rb b/lib/logstash/outputs/syslog.rb index 9b7e4b8..29fddbf 100644 --- a/lib/logstash/outputs/syslog.rb +++ b/lib/logstash/outputs/syslog.rb @@ -125,13 +125,16 @@ class LogStash::Outputs::Syslog < LogStash::Outputs::Base # syslog message format: you can choose between rfc3164 or rfc5424 config :rfc, :validate => ["rfc3164", "rfc5424"], :default => "rfc3164" + # RFC5424 structured data. + config :structured_data, :validate => :string, :default => nil + def register @client_socket = nil if ssl? @ssl_context = setup_ssl end - + if @codec.class.name == "LogStash::Codecs::Plain" if @codec.config["format"].nil? @codec = LogStash::Codecs::Plain.new({"format" => @message}) @@ -141,6 +144,11 @@ def register # use instance variable to avoid string comparison for each event @is_rfc3164 = (@rfc == "rfc3164") + + if @is_rfc3164 && !@structured_data.nil? + raise LogStash::ConfigurationError, "Structured data is not supported for RFC3164" + end + end def receive(event) @@ -169,8 +177,9 @@ def publish(event, payload) syslog_msg = "<#{priority.to_s}>#{timestamp} #{sourcehost} #{appname}[#{procid}]: #{message}" else msgid = event.sprintf(@msgid) + sd = @structured_data.nil? ? "-" : event.sprintf(@structured_data) timestamp = event.sprintf("%{+YYYY-MM-dd'T'HH:mm:ss.SSSZZ}") - syslog_msg = "<#{priority.to_s}>1 #{timestamp} #{sourcehost} #{appname} #{procid} #{msgid} - #{message}" + syslog_msg = "<#{priority.to_s}>1 #{timestamp} #{sourcehost} #{appname} #{procid} #{msgid} #{sd} #{message}" end begin diff --git a/spec/outputs/syslog_spec.rb b/spec/outputs/syslog_spec.rb index bd2b7d4..b1c7061 100644 --- a/spec/outputs/syslog_spec.rb +++ b/spec/outputs/syslog_spec.rb @@ -139,4 +139,28 @@ it_behaves_like "syslog output" end + + context "structured data is not supported for RFC3164" do + let(:options) { {"host" => "foo", "port" => "123", "rfc" => "rfc3164", "structured_data" => "[foo@12345]" } } + + it "should raise exception" do + expect { subject.register }.to raise_error(LogStash::ConfigurationError) + end + end + + context "send with both structured data and message" do + let(:options) { {"host" => "foo", "port" => "123", "rfc" => "rfc5424", "structured_data" => '[exampleSDID@32473 iut="3" eventSource="Application" eventID="1011"][examplePriority@32473 class="high"]' } } + let(:output) { /^<13>1 #{RFC3339_DATE_TIME_REGEX} baz LOGSTASH - - \[exampleSDID@32473 iut="3" eventSource="Application" eventID="1011"\]\[examplePriority@32473 class="high"\] bar\n/m } + + it_behaves_like "syslog output" + end + + context "set structured data elements from event" do + let(:event) { LogStash::Event.new({"message" => "bar", "host" => "baz", "pod" => "mypod" }) } + let(:options) { {"host" => "foo", "port" => "123", "rfc" => "rfc5424", "structured_data" => '[exampleSDID@32473 pod="%{pod}"]' } } + let(:output) { /^<13>1 #{RFC3339_DATE_TIME_REGEX} baz LOGSTASH - - \[exampleSDID@32473 pod="mypod"\] bar\n/m } + + it_behaves_like "syslog output" + end + end From 3fa526cd11e58eb673fb7b8c42dcb8ee3f2e468d Mon Sep 17 00:00:00 2001 From: Tero Saarni Date: Fri, 25 Aug 2023 11:01:29 +0300 Subject: [PATCH 2/3] Use empty string for no structured data Signed-off-by: Tero Saarni --- docs/index.asciidoc | 2 +- lib/logstash/outputs/syslog.rb | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index e824d7f..3228ca5 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -239,7 +239,7 @@ use priority field if set to false ===== `structured_data` * Value type is <> - * Default value is `nil` + * There is no default value for this setting. RFC5424 structured data is a string of one or more structured data elements, including brackets. The elements need to be formatted according to link:https://datatracker.ietf.org/doc/html/rfc5424#section-6.3[RFC5424 section 6.3], for example: diff --git a/lib/logstash/outputs/syslog.rb b/lib/logstash/outputs/syslog.rb index 29fddbf..d3ae6ca 100644 --- a/lib/logstash/outputs/syslog.rb +++ b/lib/logstash/outputs/syslog.rb @@ -126,7 +126,7 @@ class LogStash::Outputs::Syslog < LogStash::Outputs::Base config :rfc, :validate => ["rfc3164", "rfc5424"], :default => "rfc3164" # RFC5424 structured data. - config :structured_data, :validate => :string, :default => nil + config :structured_data, :validate => :string, :default => "" def register @client_socket = nil @@ -145,7 +145,7 @@ def register # use instance variable to avoid string comparison for each event @is_rfc3164 = (@rfc == "rfc3164") - if @is_rfc3164 && !@structured_data.nil? + if @is_rfc3164 && !@structured_data.empty? raise LogStash::ConfigurationError, "Structured data is not supported for RFC3164" end @@ -177,7 +177,7 @@ def publish(event, payload) syslog_msg = "<#{priority.to_s}>#{timestamp} #{sourcehost} #{appname}[#{procid}]: #{message}" else msgid = event.sprintf(@msgid) - sd = @structured_data.nil? ? "-" : event.sprintf(@structured_data) + sd = @structured_data.empty? ? "-" : event.sprintf(@structured_data) timestamp = event.sprintf("%{+YYYY-MM-dd'T'HH:mm:ss.SSSZZ}") syslog_msg = "<#{priority.to_s}>1 #{timestamp} #{sourcehost} #{appname} #{procid} #{msgid} #{sd} #{message}" end From ab1431bd83284bd415b67dd2eb273d7578d7be50 Mon Sep 17 00:00:00 2001 From: Tero Saarni Date: Fri, 25 Aug 2023 13:30:08 +0300 Subject: [PATCH 3/3] fixed review comments Signed-off-by: Tero Saarni --- docs/index.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 3228ca5..fee925f 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -244,8 +244,8 @@ use priority field if set to false RFC5424 structured data is a string of one or more structured data elements, including brackets. The elements need to be formatted according to link:https://datatracker.ietf.org/doc/html/rfc5424#section-6.3[RFC5424 section 6.3], for example: -[source,ruby] - `[exampleSDID@32473 iut="3" eventSource="Application" eventID="1011"][examplePriority@32473 class="high"]` +["source",subs="attributes"] + `[exampleSDID@32473 iut="3" eventSource="Application" eventID="1011"][examplePriority@32473 class="high"]` The new value can include `%{foo}` strings to help you build a new value from other parts of the event.