WIP: openai poc

package/etc/conf.d/conflib/_splunk/ai_parser.conf

@@ -0,0 +1,5 @@
+parser p_ai_importance_parser {
+    python(
+        class("parser_ai.AIImportanceParser")
+    );
+};

package/etc/conf.d/conflib/_splunk/splunkfields.conf

@@ -12,6 +12,7 @@ rewrite r_set_splunk_default {
             set($LOGHOST, value("fields.sc4s_container") condition(match('container' template('`SC4S_DEST_SPLUNK_INDEXED_FIELDS`') type(string) flags(substring)) ));
             set($SOURCEIP, value("fields.sc4s_fromhostip") condition(match('fromhostip' template('`SC4S_DEST_SPLUNK_INDEXED_FIELDS`') type(string) flags(substring)) ));
             set($R_UNIXTIME, value("fields.sc4s_recv_time") condition(match('r_unixtime' template('`SC4S_DEST_SPLUNK_INDEXED_FIELDS`') type(string) flags(substring)) ));
+            set($IMPORTANCE, value("fields.sc4s_importance") condition(match('importance' template('`SC4S_DEST_SPLUNK_INDEXED_FIELDS`') type(string) flags(substring)) ));
         };
 
         if ("$PROTO" == 6) {

package/etc/conf.d/sources/source_syslog/plugin.jinja

@@ -301,6 +301,8 @@ source s_{{ port_id }} {
             };
         {%- endif %}
 
+        parser(p_ai_importance_parser);
+
         rewrite {
             set($FACILITY, value("fields.sc4s_syslog_facility") condition(match('facility' template('`SC4S_DEST_SPLUNK_INDEXED_FIELDS`') type(string) flags(substring))));
             set($LEVEL, value("fields.sc4s_syslog_severity") condition(match('severity' template('`SC4S_DEST_SPLUNK_INDEXED_FIELDS`') type(string) flags(substring)) ));
@@ -500,6 +502,8 @@ source s_{{ port_id }} {
             };
         {%- endif %}
 
+        parser(p_ai_importance_parser);
+
         rewrite {
             set($FACILITY, value("fields.sc4s_syslog_facility") condition(match('facility' template('`SC4S_DEST_SPLUNK_INDEXED_FIELDS`') type(string) flags(substring))));
             set($LEVEL, value("fields.sc4s_syslog_severity") condition(match('severity' template('`SC4S_DEST_SPLUNK_INDEXED_FIELDS`') type(string) flags(substring)) ));

package/etc/pylib/parser_ai.py

@@ -0,0 +1,44 @@
+from openai import OpenAI
+
+try:
+    import syslogng
+    from syslogng import LogParser
+except Exception:
+    class LogParser:
+        pass
+
+
+class AIImportanceParser(LogParser):
+    def init(self, options):
+        self.client = OpenAI()
+        return True
+
+    def parse(self, log_message):
+        """
+        Label logs as `critical`, `important`, `neutral`, or `noise`.
+        """
+        try:
+            completion = self.client.chat.completions.create(
+                model="gpt-4o-mini",
+                messages=[
+                    {
+                        "role": "system",
+                        "content": "You are a log importance tagger. You can return only one of the following tags: `critical`, `important`, `neutral`, or `noise`. Always return a tag and only one tag."
+                    },
+                    {
+                        "role": "user",
+                        "content": f"{log_message['MESSAGE']}"
+                    }
+                ]
+            )
+
+            tag = completion.choices[0].message.content
+            if tag not in ['critical', 'important', 'neutral', 'noise']:
+                return False
+            log_message['IMPORTANCE'] = tag
+
+        except:
+            return False
+
+        # return True, other way message is dropped
+        return True

package/sbin/entrypoint.sh

@@ -18,7 +18,7 @@ export SC4S_CLEAR_NAME_CACHE=${SC4S_CLEAR_NAME_CACHE:=no}
 
 export SC4S_DEFAULT_TIMEZONE=${SC4S_DEFAULT_TIMEZONE:=GMT}
 export SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL_SECONDS=${SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL_SECONDS:=2}
-export SC4S_DEST_SPLUNK_INDEXED_FIELDS=${SC4S_DEST_SPLUNK_INDEXED_FIELDS:=r_unixtime,facility,container,loghost,destport,fromhostip,proto,severity}
+export SC4S_DEST_SPLUNK_INDEXED_FIELDS=${SC4S_DEST_SPLUNK_INDEXED_FIELDS:=r_unixtime,facility,container,loghost,destport,fromhostip,proto,severity,importance}
 
 export SC4S_OPTION_FORTINET_SOURCETYPE_PREFIX=${SC4S_OPTION_FORTINET_SOURCETYPE_PREFIX:=fgt}