From 5bfdea82f185e1f96686802afe3bfbc2299b10cd Mon Sep 17 00:00:00 2001 From: William Yardley Date: Thu, 7 Apr 2016 12:05:40 -0700 Subject: [PATCH 1/2] switch USER to HTTPDUSER for "auth" field (match email addresses) --- patterns/grok-patterns | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/patterns/grok-patterns b/patterns/grok-patterns index ed2ed487..950d5c77 100644 --- a/patterns/grok-patterns +++ b/patterns/grok-patterns @@ -90,7 +90,7 @@ QS %{QUOTEDSTRING} # Log formats SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}: -COMMONAPACHELOG %{IPORHOST:clientip} %{HTTPDUSER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-) +COMMONAPACHELOG %{IPORHOST:clientip} %{HTTPDUSER:ident} %{HTTPDUSER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-) COMBINEDAPACHELOG %{COMMONAPACHELOG} %{QS:referrer} %{QS:agent} HTTPD20_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{LOGLEVEL:loglevel}\] (?:\[client %{IPORHOST:clientip}\] ){0,1}%{GREEDYDATA:errormsg} HTTPD24_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{WORD:module}:%{LOGLEVEL:loglevel}\] \[pid %{POSINT:pid}:tid %{NUMBER:tid}\]( \(%{POSINT:proxy_errorcode}\)%{DATA:proxy_errormessage}:)?( \[client %{IPORHOST:client}:%{POSINT:clientport}\])? %{DATA:errorcode}: %{GREEDYDATA:message} From d9a220b0b3086cc5ccd134c5d67a531978df5c05 Mon Sep 17 00:00:00 2001 From: William Yardley Date: Thu, 7 Apr 2016 23:20:29 -0700 Subject: [PATCH 2/2] add simple test case (untested!) for an email address in the auth field --- spec/patterns/core_spec.rb | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/spec/patterns/core_spec.rb b/spec/patterns/core_spec.rb index a644b1c9..16bdd19f 100644 --- a/spec/patterns/core_spec.rb +++ b/spec/patterns/core_spec.rb @@ -22,10 +22,24 @@ describe "COMMONAPACHELOG" do - let(:value) { '83.149.9.216 - - [24/Feb/2015:23:13:42 +0000] "GET /presentations/logstash-monitorama-2013/images/kibana-search.png HTTP/1.1" 200 203023 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36'} + context "Typical test case" do + + let(:value) { '83.149.9.216 - - [24/Feb/2015:23:13:42 +0000] "GET /presentations/logstash-monitorama-2013/images/kibana-search.png HTTP/1.1" 200 203023 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36'} + + it "generates the clientip field" do + expect(grok_match(subject, value)).to include("clientip" => "83.149.9.216") + end + + end + + context "Email address in auth field" do + + let(:value) { '10.0.0.1 - username@example.com [07/Apr/2016:18:42:24 +0000] "GET /bar/foo/users/1/username%40example.com/authenticate?token=blargh&client_id=15 HTTP/1.1" 400 75 "" "Mozilla/5.0 (iPad; CPU OS 9_3_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13E238 Safari/601.1"'} + + it "generates the clientip field" do + expect(grok_match(subject, value)).to include("auth" => "username@example.com") + end - it "generates the clientip field" do - expect(grok_match(subject, value)).to include("clientip" => "83.149.9.216") end end