From 4b5c9d6ec1318aeb399147f11992b4d1e8f4da6c Mon Sep 17 00:00:00 2001 From: Szymon Bylica <142112966+sbylica-splunk@users.noreply.github.com> Date: Mon, 11 Aug 2025 14:16:23 +0200 Subject: [PATCH 1/3] Added docs for EKS deployment (#2785) * Added docs for EKS deployment * Changed some examples in Ansible documentation --- ansible/resources/env_file | 2 +- docs/gettingstarted/ansible-docker-podman.md | 8 +- docs/gettingstarted/ansible-docker-swarm.md | 2 +- docs/gettingstarted/eks.md | 106 +++++++++++++++++++ docs/resources/docker/sc4s_deployment.yaml | 85 +++++++++++++++ mkdocs.yml | 2 + 6 files changed, 199 insertions(+), 6 deletions(-) create mode 100644 docs/gettingstarted/eks.md create mode 100644 docs/resources/docker/sc4s_deployment.yaml diff --git a/ansible/resources/env_file b/ansible/resources/env_file index 9f73a6d46c..3eb4275e4d 100644 --- a/ansible/resources/env_file +++ b/ansible/resources/env_file @@ -1,4 +1,4 @@ -SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=http://xxx.xxx.xxx.xxx:8088 +SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://xxx.xxx.xxx.xxx:8088 SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=xxxxxxxxxxxxxxxxxx #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY=no \ No newline at end of file diff --git a/docs/gettingstarted/ansible-docker-podman.md b/docs/gettingstarted/ansible-docker-podman.md index 3f890ffa7a..1de2f6fd6b 100644 --- a/docs/gettingstarted/ansible-docker-podman.md +++ b/docs/gettingstarted/ansible-docker-podman.md @@ -23,16 +23,16 @@ docker exec -it ansible_sc4s /bin/bash * To authenticate with username and password: ``` bash -ansible-playbook -i path/to/inventory.yaml -u --ask-pass path/to/playbooks/docker.yml +ansible-playbook -i ansible/inventory/inventory.yaml -u --ask-pass ansible/playbooks/playbooks/docker.yml or -ansible-playbook -i path/to/inventory.yaml -u --ask-pass path/to/playbooks/podman.yml +ansible-playbook -i ansible/inventory/inventory.yaml -u --ask-pass ansible/playbooks/playbooks/podman.yml ``` * To authenticate using a key pair: ``` bash -ansible-playbook -i path/to/inventory.yaml -u --key-file path/to/playbooks/docker.yml +ansible-playbook -i ansible/inventory/inventory.yaml -u --key-file ansible/playbooks/playbooks/docker.yml or -ansible-playbook -i path/to/inventory.yaml -u --key-file path/to/playbooks/podman.yml +ansible-playbook -i ansible/inventory/inventory.yaml -u --key-file ansible/playbooks/playbooks/podman.yml ``` # Step 3: Validate your configuration diff --git a/docs/gettingstarted/ansible-docker-swarm.md b/docs/gettingstarted/ansible-docker-swarm.md index a34f4c1336..39417f3d96 100644 --- a/docs/gettingstarted/ansible-docker-swarm.md +++ b/docs/gettingstarted/ansible-docker-swarm.md @@ -51,7 +51,7 @@ ansible-playbook -i path/to/inventory_swarm.yaml -u --key-file Date: Thu, 21 Aug 2025 10:54:53 +0200 Subject: [PATCH 2/3] Fixed sonarqube issues (#2789) --- package/Dockerfile | 58 ++++++++-------- package/Dockerfile.enterprise | 65 ++++++++--------- package/Dockerfile.lite | 58 ++++++++-------- .../log_paths/2/lp_dest_alts_global/plugin.py | 7 +- .../conf.d/sources/source_syslog/plugin.py | 6 +- package/enterprise/etc/pylib/parser_cef.py | 5 +- .../enterprise/etc/pylib/parser_fix_dns.py | 5 +- package/enterprise/etc/pylib/parser_kvqf.py | 2 +- package/enterprise/etc/pylib/parser_leef.py | 69 +++++++++++-------- .../etc/pylib/parser_stealthbits.py | 10 +-- .../enterprise/etc/pylib/parser_vps_cache.py | 4 -- package/etc/pylib/parser_cef.py | 2 +- package/etc/pylib/parser_kvqf.py | 2 +- package/etc/pylib/parser_stealthbits.py | 1 - 14 files changed, 148 insertions(+), 146 deletions(-) diff --git a/package/Dockerfile b/package/Dockerfile index c51d065327..c61bb433b2 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -21,35 +21,35 @@ FROM ghcr.io/axoflow/axosyslog:${SYSLOGNG_VERSION} RUN apk add -U --upgrade --no-cache \ - bash \ - libxml2 \ - expat \ - binutils \ - musl \ - build-base \ - curl \ - grep \ - less \ - net-tools \ - netcat-openbsd \ - openssl \ - postgresql-libs \ - procps \ - py3-pip \ - python3 \ - python3-dev \ - libffi-dev \ - shadow \ - socat \ - tzdata \ - wget \ - cargo \ - ca-certificates \ - poetry \ - gdb \ - py3-poetry-plugin-export \ - py3-virtualenv \ - xz-libs \ + bash \ + binutils \ + build-base \ + ca-certificates \ + cargo \ + curl \ + expat \ + gdb \ + grep \ + less \ + libffi-dev \ + libxml2 \ + musl \ + net-tools \ + netcat-openbsd \ + openssl \ + poetry \ + postgresql-libs \ + procps \ + py3-pip \ + py3-poetry-plugin-export \ + py3-virtualenv \ + python3 \ + python3-dev \ + shadow \ + socat \ + tzdata \ + wget \ + xz-libs \ && groupadd --gid 1024 syslog \ && useradd -M -g 1024 -u 1024 syslog \ && usermod -L syslog \ diff --git a/package/Dockerfile.enterprise b/package/Dockerfile.enterprise index 93823cb1f3..40cb5e7849 100644 --- a/package/Dockerfile.enterprise +++ b/package/Dockerfile.enterprise @@ -21,35 +21,35 @@ FROM ghcr.io/axoflow/axosyslog:${SYSLOGNG_VERSION} RUN apk add -U --upgrade --no-cache \ - bash \ - libxml2 \ - expat \ - binutils \ - musl \ - build-base \ - curl \ - grep \ - less \ - net-tools \ - netcat-openbsd \ - openssl \ - postgresql-libs \ - procps \ - py3-pip \ - python3 \ - python3-dev \ - libffi-dev \ - shadow \ - socat \ - tzdata \ - wget \ - cargo \ - ca-certificates \ - poetry \ - gdb \ - py3-poetry-plugin-export \ - py3-virtualenv \ - xz-libs \ + bash \ + binutils \ + build-base \ + ca-certificates \ + cargo \ + curl \ + expat \ + gdb \ + grep \ + less \ + libffi-dev \ + libxml2 \ + musl \ + net-tools \ + netcat-openbsd \ + openssl \ + poetry \ + postgresql-libs \ + procps \ + py3-pip \ + py3-poetry-plugin-export \ + py3-virtualenv \ + python3 \ + python3-dev \ + shadow \ + socat \ + tzdata \ + wget \ + xz-libs \ && groupadd --gid 1024 syslog \ && useradd -M -g 1024 -u 1024 syslog \ && usermod -L syslog \ @@ -64,13 +64,14 @@ EXPOSE 6514/tcp #/dev/log a low priv user cannot read this and the container will fail in SC4S #and other uses the low user may be selected -HEALTHCHECK --interval=2m --timeout=5s --start-period=30s CMD /usr/sbin/syslog-ng-ctl healthcheck --timeout 5 +HEALTHCHECK --interval=2m --timeout=5s --start-period=30s CMD ["/usr/sbin/syslog-ng-ctl", "healthcheck", "--timeout", "5"] COPY pyproject.toml / COPY poetry.lock / RUN python3 -m venv /var/lib/python-venv \ - && poetry export --format requirements.txt --without-hashes | /var/lib/python-venv/bin/pip3 --no-cache-dir install -r /dev/stdin \ + && poetry export --format requirements.txt --without-hashes \ + | /var/lib/python-venv/bin/pip3 --no-cache-dir install -r /dev/stdin \ && /var/lib/python-venv/bin/pip3 install --no-cache-dir --upgrade tornado==6.4.2 \ && apk del build-base python3-dev libffi-dev @@ -87,7 +88,7 @@ COPY package/sbin/source_ports_validator.py / ENV SC4S_CONTAINER_OPTS=--no-caps ARG VERSION=unknown -RUN echo $VERSION>/etc/syslog-ng/VERSION +RUN echo "$VERSION">/etc/syslog-ng/VERSION ENTRYPOINT ["/entrypoint.sh"] diff --git a/package/Dockerfile.lite b/package/Dockerfile.lite index f55a657882..61301205f9 100644 --- a/package/Dockerfile.lite +++ b/package/Dockerfile.lite @@ -21,35 +21,35 @@ FROM ghcr.io/axoflow/axosyslog:${SYSLOGNG_VERSION} RUN apk add -U --upgrade --no-cache \ - bash \ - libxml2 \ - expat \ - binutils \ - musl \ - build-base \ - curl \ - grep \ - less \ - net-tools \ - netcat-openbsd \ - openssl \ - postgresql-libs \ - procps \ - py3-pip \ - python3 \ - python3-dev \ - libffi-dev \ - shadow \ - socat \ - tzdata \ - wget \ - cargo \ - ca-certificates \ - poetry \ - gdb \ - py3-poetry-plugin-export \ - py3-virtualenv \ - xz-libs \ + bash \ + binutils \ + build-base \ + ca-certificates \ + cargo \ + curl \ + expat \ + gdb \ + grep \ + less \ + libffi-dev \ + libxml2 \ + musl \ + net-tools \ + netcat-openbsd \ + openssl \ + poetry \ + postgresql-libs \ + procps \ + py3-pip \ + py3-poetry-plugin-export \ + py3-virtualenv \ + python3 \ + python3-dev \ + shadow \ + socat \ + tzdata \ + wget \ + xz-libs \ && groupadd --gid 1024 syslog \ && useradd -M -g 1024 -u 1024 syslog \ && usermod -L syslog \ diff --git a/package/enterprise/etc/conf.d/log_paths/2/lp_dest_alts_global/plugin.py b/package/enterprise/etc/conf.d/log_paths/2/lp_dest_alts_global/plugin.py index 13bb14603d..76081f7f80 100755 --- a/package/enterprise/etc/conf.d/log_paths/2/lp_dest_alts_global/plugin.py +++ b/package/enterprise/etc/conf.d/log_paths/2/lp_dest_alts_global/plugin.py @@ -36,7 +36,7 @@ def normalize_env_variable_input(env_variable: str): modev = os.environ.get(f"SC4S_DEST_SPLUNK_HEC_{r}_MODE", "GLOBAL") if ( r == "DEFAULT" - and not os.environ.get("SC4S_DEST_SPLUNK_HEC_GLOBAL", "") == "" + and os.environ.get("SC4S_DEST_SPLUNK_HEC_GLOBAL", "") != "" ): if os.environ.get("SC4S_DEST_SPLUNK_HEC_GLOBAL", "yes").lower() in [ "true", @@ -68,9 +68,8 @@ def normalize_env_variable_input(env_variable: str): if r != "": modev = os.environ.get(f"SC4S_DEST_{t}_{r}_MODE", "GLOBAL") filter = os.environ.get(f"SC4S_DEST_{t}_{r}_FILTER", "") - if filter == "": - if t == "BSD": - filter = '"${MSG}" ne ""' + if filter == "" and t == "BSD": + filter = '"${MSG}" ne ""' if modev.upper() in ("GLOBAL", "SELECT"): global_dests[r] = { "destination": f"d_{t.lower()}_{r.lower()}", diff --git a/package/enterprise/etc/conf.d/sources/source_syslog/plugin.py b/package/enterprise/etc/conf.d/sources/source_syslog/plugin.py index fdf790b0df..1d48a8a6b1 100755 --- a/package/enterprise/etc/conf.d/sources/source_syslog/plugin.py +++ b/package/enterprise/etc/conf.d/sources/source_syslog/plugin.py @@ -53,8 +53,6 @@ def normalize_env_variable_input(env_variable: str): if len(port_parts) == 2 or len(port_parts) == 3: vendor = port_parts[0].lower() product = port_parts[1].lower() - else: - pass outputText = tm.render( vendor=vendor, @@ -131,8 +129,8 @@ def normalize_env_variable_input(env_variable: str): "HIGH:!aNULL:!eNULL:!kECDH:!aDH:!RC4:!3DES:!CAMELLIA:!MD5:!PSK:!SRP:!KRB5:@STRENGTH", ), ebpf_no_sockets=int(os.getenv("SC4S_EBPF_NO_SOCKETS", 4)), - enable_parallelize=normalize_env_variable_input(f"SC4S_ENABLE_PARALLELIZE"), - parallelize_no_partitions=int(os.getenv(f"SC4S_PARALLELIZE_NO_PARTITION", 4)), + enable_parallelize=normalize_env_variable_input("SC4S_ENABLE_PARALLELIZE"), + parallelize_no_partitions=int(os.getenv("SC4S_PARALLELIZE_NO_PARTITION", 4)), set_source_sc4s=normalize_env_variable_input("SC4S_SET_SOURCE_AS_SC4S"), ) print(outputText) diff --git a/package/enterprise/etc/pylib/parser_cef.py b/package/enterprise/etc/pylib/parser_cef.py index 7d0673fd45..13e774960c 100644 --- a/package/enterprise/etc/pylib/parser_cef.py +++ b/package/enterprise/etc/pylib/parser_cef.py @@ -21,14 +21,13 @@ def parse(self, log_message): try: data = log_message.get_as_str(".metadata.cef.ext", "") - rpairs = re.findall(r"([^=\s]+)=((?:[\\]=|[^=])+)(?:\s|$)", data) + rpairs = re.findall(r"([^=\s]+)=((?:\\=|[^=])+)(?:\s|$)", data) pairs = {} keys = [] for p in rpairs: pairs[p[0]] = p[1] keys.append(p[0]) - cleanpairs = {} for k in keys: if k.endswith("Label"): vk = k.rstrip("Label") @@ -50,5 +49,5 @@ def parse(self, log_message): lines = traceback.format_exception(exc_type, exc_value, exc_traceback) self.logger.debug("".join("!! " + line for line in lines)) return False - self.logger.debug("kvqf_parse.parse complete") + return True \ No newline at end of file diff --git a/package/enterprise/etc/pylib/parser_fix_dns.py b/package/enterprise/etc/pylib/parser_fix_dns.py index de3bbd7d15..a35e30dbea 100644 --- a/package/enterprise/etc/pylib/parser_fix_dns.py +++ b/package/enterprise/etc/pylib/parser_fix_dns.py @@ -3,7 +3,6 @@ resolves IP to hostname value pair names are hard-coded """ -import re import socket try: @@ -25,7 +24,7 @@ def parse(self, log_message): try: ipaddr = log_message.get_as_str("SOURCEIP", "", repr="internal") - hostname, aliaslist, ipaddrlist = socket.gethostbyaddr(ipaddr) + hostname, _, _ = socket.gethostbyaddr(ipaddr) if hostname == ipaddr: return False @@ -51,7 +50,7 @@ def parse(self, log_message): try: ipaddr = log_message.get_as_str("SOURCEIP", "", repr="internal") - fqdn, aliaslist, ipaddrlist = socket.gethostbyaddr(ipaddr) + fqdn, _, _ = socket.gethostbyaddr(ipaddr) if fqdn == ipaddr: return False diff --git a/package/enterprise/etc/pylib/parser_kvqf.py b/package/enterprise/etc/pylib/parser_kvqf.py index c5596c55f0..b174b7b6b0 100644 --- a/package/enterprise/etc/pylib/parser_kvqf.py +++ b/package/enterprise/etc/pylib/parser_kvqf.py @@ -26,7 +26,7 @@ def parse(self, log_message): matches = re.finditer( regex, log_message.get_as_str(".tmp.pairs", ""), re.MULTILINE ) - for matchNum, match in enumerate(matches, start=1): + for _, match in enumerate(matches, start=1): k = match.groups()[0] v = match.groups()[1] log_message[f".values.{k}"] = v diff --git a/package/enterprise/etc/pylib/parser_leef.py b/package/enterprise/etc/pylib/parser_leef.py index 1279701314..5c5010cb2f 100644 --- a/package/enterprise/etc/pylib/parser_leef.py +++ b/package/enterprise/etc/pylib/parser_leef.py @@ -1,7 +1,5 @@ import re import binascii -import sys -import traceback try: import syslogng @@ -17,45 +15,64 @@ def init(self, options): self.regex = r"( ?(?:[A-Z]{2,4}T|HAEC|IDLW|MSK|NT|UTC|THA))" self.logger = syslogng.Logger() return True + + def parse_message_from_pair(self, pair, log_message): + f, v = pair.split("=", 1) + if f == "devTime": + log_message[".leef." + f] = re.sub( + self.regex, "", v, 0, re.MULTILINE + ) + else: + log_message[".leef." + f] = v - def parse(self, log_message): + def parse_v1(self, log_message, event, structure, separator): + pairs = event.split(separator) + if len(pairs) < 4: + separator = "|" + pairs = structure[5:] + event = "\t".join(pairs) + log_message[".leef.event"] = event + return event, pairs, separator + + def parse_v2(self, event, structure, separator): + # V2 messages should always provide the sep but some fail do comply + # with the format spec if they don't assume tab + if len(structure) == 6 or not structure[5]: + pairs = event.split(separator) + else: + separator = structure[5] + if separator.startswith("0"): + separator = separator[1:] + pairs = event.split(separator) + return event, pairs, separator + def parse(self, log_message): try: msg = log_message.get_as_str("MESSAGE", "") # All LEEF message are | separated super structures structure = msg.split("|") - # Indexed fields for Splunk + # Indexed fields for Splunk log_message[".metadata.leef.version"] = structure[0][5:] log_message[".metadata.leef.vendor"] = structure[1] log_message[".metadata.leef.product"] = structure[2] log_message[".metadata.leef.product_version"] = structure[3] log_message[".metadata.leef.EventID"] = structure[4] + # We just want the event field event = structure[len(structure) - 1] log_message[".leef.event"] = event + + separator = "\t" + pairs = [] + # V1 will always use tab if structure[0][5:].startswith("1"): - separator = "\t" lv = "1" - pairs = event.split(separator) - if len(pairs) < 4: - separator = "|" - pairs = structure[5:] - event = "\t".join(pairs) - log_message[".leef.event"] = event + event, pairs, separator = self.parse_v1(log_message, event, structure, separator) else: lv = "2" - # V2 messages should always provide the sep but some fail do comply - # with the format spec if they don't assume tab - if len(structure) == 6 or not structure[5]: - separator = "\t" - pairs = event.split(separator) - else: - separator = structure[5] - if separator.startswith("0"): - separator = separator[1:] - pairs = event.split(separator) + event, pairs, separator = self.parse_v2(event, structure, separator) if separator.startswith("x"): hex_sep = f"0{separator.lower()}" @@ -70,15 +87,9 @@ def parse(self, log_message): log_message["fields.sc4s_product"] = structure[2] for p in pairs: - f, v = p.split("=", 1) - if f == "devTime": - log_message[".leef." + f] = re.sub( - self.regex, "", v, 0, re.MULTILINE - ) - else: - log_message[".leef." + f] = v + self.parse_message_from_pair(p, log_message) except Exception as e: log_message[".metadata.leef.exception"] = str(e) # return True, other way message is dropped - return True \ No newline at end of file + return True diff --git a/package/enterprise/etc/pylib/parser_stealthbits.py b/package/enterprise/etc/pylib/parser_stealthbits.py index bdf0369816..89445d5119 100644 --- a/package/enterprise/etc/pylib/parser_stealthbits.py +++ b/package/enterprise/etc/pylib/parser_stealthbits.py @@ -1,7 +1,6 @@ import re try: - import syslogng from syslogng import LogParser except Exception: @@ -10,6 +9,7 @@ class LogParser: regex = r"^(.*[\.\!\?])?(.*:.*)" +alert_text_key = ".values.AlertText" class alerttext_kv(LogParser): @@ -17,13 +17,13 @@ def init(self, options): return True def parse(self, log_message): - match = re.search(regex, log_message.get_as_str(".values.AlertText", "")) + match = re.search(regex, log_message.get_as_str(alert_text_key, "")) if match: - log_message[".values.AlertText"] = match.groups()[0] + log_message[alert_text_key] = match.groups()[0] text = match.groups()[1] else: - text = log_message.get_as_str(".values.AlertText", "") - log_message[".values.AlertText"] = "" + text = log_message.get_as_str(alert_text_key, "") + log_message[alert_text_key] = "" pairs = text.split("; ") diff --git a/package/enterprise/etc/pylib/parser_vps_cache.py b/package/enterprise/etc/pylib/parser_vps_cache.py index 4c8cf21250..d545620158 100644 --- a/package/enterprise/etc/pylib/parser_vps_cache.py +++ b/package/enterprise/etc/pylib/parser_vps_cache.py @@ -95,7 +95,3 @@ def send(self, log_message): def flush(self): self.db.commit() return True - - -if __name__ == "__main__": - pass diff --git a/package/etc/pylib/parser_cef.py b/package/etc/pylib/parser_cef.py index 28fd68498e..32dede7e12 100644 --- a/package/etc/pylib/parser_cef.py +++ b/package/etc/pylib/parser_cef.py @@ -49,5 +49,5 @@ def parse(self, log_message): lines = traceback.format_exception(exc_type, exc_value, exc_traceback) self.logger.debug("".join("!! " + line for line in lines)) return False - self.logger.debug("kvqf_parse.parse complete") + return True diff --git a/package/etc/pylib/parser_kvqf.py b/package/etc/pylib/parser_kvqf.py index 99206af8d1..b7d3253681 100644 --- a/package/etc/pylib/parser_kvqf.py +++ b/package/etc/pylib/parser_kvqf.py @@ -26,7 +26,7 @@ def parse(self, log_message): matches = re.finditer( regex, log_message.get_as_str(".tmp.pairs", ""), re.MULTILINE ) - for match_num, match in enumerate(matches, start=1): + for _, match in enumerate(matches, start=1): k = match.groups()[0] v = match.groups()[1] log_message[f".values.{k}"] = v diff --git a/package/etc/pylib/parser_stealthbits.py b/package/etc/pylib/parser_stealthbits.py index 07a3c87f10..99800d479f 100644 --- a/package/etc/pylib/parser_stealthbits.py +++ b/package/etc/pylib/parser_stealthbits.py @@ -1,7 +1,6 @@ import re try: - import syslogng from syslogng import LogParser except Exception: From 25a4bac8e8c59cebb376b4ad2e6afbcf989dcef7 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 11 Sep 2025 17:39:55 +0000 Subject: [PATCH 3/3] chore(deps): update dependency splunk-sdk to v2.1.1 --- poetry.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/poetry.lock b/poetry.lock index a6b91f6899..6d4be2b96a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1455,13 +1455,13 @@ files = [ [[package]] name = "splunk-sdk" -version = "2.1.0" +version = "2.1.1" description = "The Splunk Software Development Kit for Python." optional = false python-versions = "*" groups = ["dev"] files = [ - {file = "splunk-sdk-2.1.0.tar.gz", hash = "sha256:63f9a259a7c84d0c3b0b32cae652365b03f0f926acdb894b51456005df74ae21"}, + {file = "splunk-sdk-2.1.1.tar.gz", hash = "sha256:46300d52f09e0aed7e5962ce2ba08ef54421ffb3a538c6af6164dcbf9f075faa"}, ] [package.dependencies]