diff --git a/docs/sources/vendor/Citrix/netscaler.md b/docs/sources/vendor/Citrix/netscaler.md index a758992eff..1f14a51d9a 100644 --- a/docs/sources/vendor/Citrix/netscaler.md +++ b/docs/sources/vendor/Citrix/netscaler.md @@ -7,27 +7,33 @@ ## Links -| Ref | Link | -|----------------|---------------------------------------------------------------------------------------------------------| -| Splunk Add-on | | +| Ref | Link | +|----------------|-----------------------------------------------------------------------------------------------------| +| Splunk Add-on | | | Product Manual | | ## Sourcetypes -| sourcetype | notes | -|----------------|---------------------------------------------------------------------------------------------------------| -| citrix:netscaler:syslog | None | -| citrix:netscaler:appfw | None | -| citrix:netscaler:appfw:cef | None | +| sourcetype | notes | +|----------------------------|-------| +| citrix:netscaler:syslog | None | +| citrix:netscaler:appfw | None | +| citrix:netscaler:appfw:cef | None | ## Sourcetype and Index Configuration -| key | sourcetype | index | notes | -|----------------|----------------|----------------|----------------| -| citrix_netscaler | citrix:netscaler:syslog | netfw | none | -| citrix_netscaler | citrix:netscaler:appfw | netfw | none | -| citrix_netscaler | citrix:netscaler:appfw:cef | netfw | none | +| key | sourcetype | index | notes | +|------------------|----------------------------|-------|-------| +| citrix_netscaler | citrix:netscaler:syslog | netfw | none | +| citrix_netscaler | citrix:netscaler:appfw | netfw | none | +| citrix_netscaler | citrix:netscaler:appfw:cef | netfw | none | ## Source Setup and Configuration -* Follow vendor configuration steps per Product Manual above. Ensure the data format selected is "DDMMYYYY" +* Follow vendor configuration steps per Product Manual above. + +## Options + +| Variable | default | description | +|--------------------------------------------|--------------|-----------------------------------------------------------------------------------------------| +| `SC4S_IGNORE_MMDD_LEGACY_CITRIX_NETSCALER` | empty string | (empty/yes) Set to "yes" for parsing the date in format `dd/mm/yyyy` instead of `mm/dd/yyyy`. | diff --git a/docs/sources/vendor/Dell/emc_powerstore.md b/docs/sources/vendor/Dell/emc_powerstore.md new file mode 100644 index 0000000000..bf324653e6 --- /dev/null +++ b/docs/sources/vendor/Dell/emc_powerstore.md @@ -0,0 +1,26 @@ +# Dell Powerstore + +## Key facts + +* MSG Format based filter +* Legacy BSD Format default port 514 + +## Links + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------------------------------| +| Splunk Add-on | N/A | +| Add-on Manual | N/A | +| Product Manual | [Powerstore Documentation](https://www.dell.com/support/kbdoc/en-us/000130110/powerstore-info-hub-product-documentation-videos) | + +## Sourcetypes + +| sourcetype | notes | +|-----------------------|-------| +| `dell:emc:powerstore` | None | + +### Index Configuration + +| key | sourcetype | index | notes | +|--------------------|-----------------------|----------|-------| +| dellemc_powerstore | `dell:emc:powerstore` | `netops` | none | diff --git a/docs/sources/vendor/ISC/dhcpd.md b/docs/sources/vendor/ISC/dhcpd.md index f25d615d27..538a3a1287 100644 --- a/docs/sources/vendor/ISC/dhcpd.md +++ b/docs/sources/vendor/ISC/dhcpd.md @@ -19,13 +19,13 @@ see that source documentation for instructions | sourcetype | notes | |----------------|---------------------------------------------------------------------------------------------------------| -| isc:dhcp | none | +| isc:dhcpd | none | ### Index Configuration | key | index | notes | |----------------|------------|----------------| -| isc_dhcp | isc:dhcp | none | +| isc_dhcpd | netipam | none | ### Filter type @@ -42,5 +42,5 @@ An active site will generate frequent events use the following search to check f Verify timestamp, and host values match as expected ``` -index= (sourcetype=isc:dhcp") +index= (sourcetype=isc:dhcpd") ``` diff --git a/docs/upgrade.md b/docs/upgrade.md index 813c4f1824..5a318be619 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -18,6 +18,10 @@ For a step by step guide [see here](./v3_upgrade.md). You may need to migrate legacy log paths or version 1 app-parsers for version 2. To do this, open an issue and attach the original configuration and a compressed pcap of sample data for testing. We will evaluate whether to include the source in an upcoming release. +### Upgrade from <3.37.0 +In `entrypoint.sh` the old variable mappings `SPLUNK_HEC_URL`, `SPLUNK_HEC_TOKEN`, `SC4S_DEST_SPLUNK_HEC_TLS_VERIFY` are deprecated and will not be +further reassigned, instead use `SC4S_DEST_SPLUNK_HEC_DEFAULT_URL`, `SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN`, `SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY`. + ### Upgrade from <3.33.0 In NetApp ONTAP, the ontap:ems sourcetype has been updated to netapp:ontap:audit, so old logs are now classified under netapp:ontap:audit. Additionally, a new netapp:ontap:ems sourcetype has been introduced. If you upgrade and want these new changes, ensure that you set `SC4S_NETAPP_ONTAP_NEW_FORMAT` environment variable to `yes` and configure your system to send the logs to a specific port or have a hostname-based configuration in place for proper log onboarding into Splunk. diff --git a/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslog-citrix_netscaler.conf b/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslog-citrix_netscaler.conf index 0408a0ff77..a5316cc179 100644 --- a/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslog-citrix_netscaler.conf +++ b/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslog-citrix_netscaler.conf @@ -10,7 +10,7 @@ block parser app-almost-syslog-citrix_netscaler() { parser { regexp-parser( prefix(".tmp.") - patterns('^(?\<\d+>) (?(?\d\d)\/\d\d\/\d\d\d\d:\d\d:\d\d:\d\d ?(?\w+))? (?[^ ]+) (?[A-Z\-0-9]+ : .*)') + patterns('^(?\<\d+\>) (?(?\d\d)\/\d\d\/\d\d\d\d:\d\d:\d\d:\d\d ?(?\w+))? (?[^ ]+) (?[A-Z\-0-9]+ : .*)') ); }; parser { @@ -19,11 +19,12 @@ block parser app-almost-syslog-citrix_netscaler() { ); }; + if { - filter { "${.tmp.tspart1}" eq "$R_DAY"}; + filter { "`SC4S_IGNORE_MMDD_LEGACY_CITRIX_NETSCALER`" eq "yes" or "${.tmp.tspart1}" eq "${DAY}"}; parser { date-parser-nofilter( - format('%d/%m/%Y:%H:%M:%S %z','%d/%m/%Y:%H:%M:%S') + format('%d/%m/%Y:%H:%M:%S %z','%d/%m/%Y:%H:%M:%S','%d/%m/%Y:%H:%M:%S %Z') template("${.tmp.timestamp}") ); }; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-dell_powerstore.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-dell_powerstore.conf new file mode 100644 index 0000000000..7a3c0044b8 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-dell_powerstore.conf @@ -0,0 +1,18 @@ +block parser app-syslog-dell_powerstore() { + channel { + rewrite { + r_set_splunk_dest_default( + index('netops') + sourcetype('dell:emc:powerstore') + vendor('dellemc') + product('powerstore') + ); + }; + }; +}; +application app-syslog-dell_powerstore[sc4s-network-source] { + filter { + match('\[PowerStore_audit_event@1139' value("MESSAGE")); + }; + parser { app-syslog-dell_powerstore(); }; +}; \ No newline at end of file diff --git a/package/etc/conf.d/conflib/almost-syslog/app-almost-syslog-citrix_netscaler.conf b/package/etc/conf.d/conflib/almost-syslog/app-almost-syslog-citrix_netscaler.conf index 0408a0ff77..a5316cc179 100644 --- a/package/etc/conf.d/conflib/almost-syslog/app-almost-syslog-citrix_netscaler.conf +++ b/package/etc/conf.d/conflib/almost-syslog/app-almost-syslog-citrix_netscaler.conf @@ -10,7 +10,7 @@ block parser app-almost-syslog-citrix_netscaler() { parser { regexp-parser( prefix(".tmp.") - patterns('^(?\<\d+>) (?(?\d\d)\/\d\d\/\d\d\d\d:\d\d:\d\d:\d\d ?(?\w+))? (?[^ ]+) (?[A-Z\-0-9]+ : .*)') + patterns('^(?\<\d+\>) (?(?\d\d)\/\d\d\/\d\d\d\d:\d\d:\d\d:\d\d ?(?\w+))? (?[^ ]+) (?[A-Z\-0-9]+ : .*)') ); }; parser { @@ -19,11 +19,12 @@ block parser app-almost-syslog-citrix_netscaler() { ); }; + if { - filter { "${.tmp.tspart1}" eq "$R_DAY"}; + filter { "`SC4S_IGNORE_MMDD_LEGACY_CITRIX_NETSCALER`" eq "yes" or "${.tmp.tspart1}" eq "${DAY}"}; parser { date-parser-nofilter( - format('%d/%m/%Y:%H:%M:%S %z','%d/%m/%Y:%H:%M:%S') + format('%d/%m/%Y:%H:%M:%S %z','%d/%m/%Y:%H:%M:%S','%d/%m/%Y:%H:%M:%S %Z') template("${.tmp.timestamp}") ); }; diff --git a/package/etc/conf.d/conflib/syslog/app-syslog-dell_powerstore.conf b/package/etc/conf.d/conflib/syslog/app-syslog-dell_powerstore.conf new file mode 100644 index 0000000000..7a3c0044b8 --- /dev/null +++ b/package/etc/conf.d/conflib/syslog/app-syslog-dell_powerstore.conf @@ -0,0 +1,18 @@ +block parser app-syslog-dell_powerstore() { + channel { + rewrite { + r_set_splunk_dest_default( + index('netops') + sourcetype('dell:emc:powerstore') + vendor('dellemc') + product('powerstore') + ); + }; + }; +}; +application app-syslog-dell_powerstore[sc4s-network-source] { + filter { + match('\[PowerStore_audit_event@1139' value("MESSAGE")); + }; + parser { app-syslog-dell_powerstore(); }; +}; \ No newline at end of file diff --git a/package/lite/etc/addons/citrix/app-almost-syslog-citrix_netscaler.conf b/package/lite/etc/addons/citrix/app-almost-syslog-citrix_netscaler.conf index 0408a0ff77..a5316cc179 100644 --- a/package/lite/etc/addons/citrix/app-almost-syslog-citrix_netscaler.conf +++ b/package/lite/etc/addons/citrix/app-almost-syslog-citrix_netscaler.conf @@ -10,7 +10,7 @@ block parser app-almost-syslog-citrix_netscaler() { parser { regexp-parser( prefix(".tmp.") - patterns('^(?\<\d+>) (?(?\d\d)\/\d\d\/\d\d\d\d:\d\d:\d\d:\d\d ?(?\w+))? (?[^ ]+) (?[A-Z\-0-9]+ : .*)') + patterns('^(?\<\d+\>) (?(?\d\d)\/\d\d\/\d\d\d\d:\d\d:\d\d:\d\d ?(?\w+))? (?[^ ]+) (?[A-Z\-0-9]+ : .*)') ); }; parser { @@ -19,11 +19,12 @@ block parser app-almost-syslog-citrix_netscaler() { ); }; + if { - filter { "${.tmp.tspart1}" eq "$R_DAY"}; + filter { "`SC4S_IGNORE_MMDD_LEGACY_CITRIX_NETSCALER`" eq "yes" or "${.tmp.tspart1}" eq "${DAY}"}; parser { date-parser-nofilter( - format('%d/%m/%Y:%H:%M:%S %z','%d/%m/%Y:%H:%M:%S') + format('%d/%m/%Y:%H:%M:%S %z','%d/%m/%Y:%H:%M:%S','%d/%m/%Y:%H:%M:%S %Z') template("${.tmp.timestamp}") ); }; diff --git a/package/lite/etc/addons/dell/app-syslog-dell_powerstore.conf b/package/lite/etc/addons/dell/app-syslog-dell_powerstore.conf new file mode 100644 index 0000000000..7a3c0044b8 --- /dev/null +++ b/package/lite/etc/addons/dell/app-syslog-dell_powerstore.conf @@ -0,0 +1,18 @@ +block parser app-syslog-dell_powerstore() { + channel { + rewrite { + r_set_splunk_dest_default( + index('netops') + sourcetype('dell:emc:powerstore') + vendor('dellemc') + product('powerstore') + ); + }; + }; +}; +application app-syslog-dell_powerstore[sc4s-network-source] { + filter { + match('\[PowerStore_audit_event@1139' value("MESSAGE")); + }; + parser { app-syslog-dell_powerstore(); }; +}; \ No newline at end of file diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh index daadbde39e..4b6de63e71 100755 --- a/package/sbin/entrypoint.sh +++ b/package/sbin/entrypoint.sh @@ -1,13 +1,15 @@ #!/usr/bin/env bash function join_by { local d=$1; shift; local f=$1; shift; printf %s "$f" "${@/#/$d}"; } + +# Activate python environment and run parsing/caching for conf files . /var/lib/python-venv/bin/activate export PYTHONPATH=/etc/syslog-ng/pylib - python3 /etc/syslog-ng/pylib/parser_source_cache.py +# Configuring environment variables export SC4S_LISTEN_STATUS_PORT=${SC4S_LISTEN_STATUS_PORT:=8080} -# These path variables allow for a single entrypoint script to be utilized for both Container and BYOE runtimes + export SC4S_LISTEN_DEFAULT_TCP_PORT=${SC4S_LISTEN_DEFAULT_TCP_PORT:=514} export SC4S_LISTEN_DEFAULT_UDP_PORT=${SC4S_LISTEN_DEFAULT_UDP_PORT:=514} export SC4S_LISTEN_DEFAULT_TLS_PORT=${SC4S_LISTEN_DEFAULT_TLS_PORT:=6514} @@ -22,20 +24,19 @@ export SC4S_DEST_SPLUNK_INDEXED_FIELDS=${SC4S_DEST_SPLUNK_INDEXED_FIELDS:=r_unix export SC4S_OPTION_FORTINET_SOURCETYPE_PREFIX=${SC4S_OPTION_FORTINET_SOURCETYPE_PREFIX:=fgt} -if [ -n "${SPLUNK_HEC_URL}" ]; then export SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=$SPLUNK_HEC_URL; fi -if [ -n "${SPLUNK_HEC_TOKEN}" ]; then export SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=$SPLUNK_HEC_TOKEN; fi -if [ -n "${SC4S_DEST_SPLUNK_HEC_TLS_VERIFY}" ]; then export SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY=$SC4S_DEST_SPLUNK_HEC_TLS_VERIFY; fi - +# Variables with path to sc4s directories +# These path variables allow for a single entrypoint script to be utilized for both Container and BYOE runtimes export SC4S_ETC=${SC4S_ETC:=/etc/syslog-ng} export SC4S_TLS=${SC4S_TLS:=/etc/syslog-ng/tls} export SC4S_VAR=${SC4S_VAR:=/var/lib/syslog-ng} export SC4S_BIN=${SC4S_BIN:=/usr/bin} export SC4S_SBIN=${SC4S_SBIN:=/usr/sbin} +# Set list with alternate destinations than HEC export SC4S_DESTS_FILTERED_ALTERNATES=$(env | grep _FILTERED_ALTERNATES= | grep -v SC4S_DEST_GLOBAL_FILTERED_ALTERNATES | cut -d= -f2 | sort | uniq | paste -s -d, -) [ -z "$SC4S_DESTS_FILTERED_ALTERNATES" ] && unset SC4S_DESTS_FILTERED_ALTERNATES -# SIGTERM-handler +# SIGTERM(15) - requests termination (default signal for kill) term_handler() { # SIGTERM on valid PID; return exit code 0 (clean exit) if [ $pid -ne 0 ]; then @@ -48,7 +49,7 @@ term_handler() { exit 143 } -# SIGHUP-handler +# SIGHUP(1) - used to reload configs or restart processes hup_handler() { if [ $pid -ne 0 ]; then echo Reloading syslog-ng... @@ -56,7 +57,7 @@ hup_handler() { fi } -# SIGQUIT-handler +# SIGQUIT(3) - used on process to quit and dump core quit_handler() { if [ $pid -ne 0 ]; then echo Quitting syslog-ng... @@ -65,10 +66,39 @@ quit_handler() { fi } +# SIGABRT(6) - abort signal +abrt_handler() { +# SIGABRT on valid PID + if [ $pid -ne 0 ]; then + echo Aborting syslog-ng... + kill -SIGABRT ${pid} + wait ${pid} + exit $? + fi +# 128 + 6 + exit 134 +} + +# SIGINT(2) - interrupts the process (ex. Ctrl+C) +int_handler() { + if [ $pid -ne 0 ]; then + echo Interupting syslog-ng... + kill -SIGINT ${pid} + wait ${pid} + exit $? + fi +# 128 + 2 + exit 130 +} + +# Setting traps to run handler function based on received signal trap 'kill ${!}; hup_handler' SIGHUP trap 'kill ${!}; term_handler' SIGTERM trap 'kill ${!}; quit_handler' SIGQUIT +trap 'kill ${!}; abrt_handler' SIGABRT +trap 'kill ${!}; int_handler' SIGINT +# Create directories needed for SC4S mkdir -p $SC4S_VAR/log/ mkdir -p $SC4S_ETC/conf.d/local/context/ mkdir -p $SC4S_ETC/conf.d/merged/context/ @@ -80,6 +110,7 @@ mkdir -p $SC4S_ETC/addons/ # copy all files in context_templates to conf.d/local/context cp -f $SC4S_ETC/context_templates/* $SC4S_ETC/conf.d/local/context +# Copying the config files from sc4s repository to sc4s local directory # check if runtime environment is k8s if [ "$SC4S_RUNTIME_ENV" == "k8s" ] then @@ -89,10 +120,7 @@ then mkdir -p $SC4S_ETC/conf.d/configmap/addons/ # copy all files in configmap/context to conf.d/local/context - cp -R -f $SC4S_ETC/conf.d/configmap/* $SC4S_ETC/conf.d/local/ - #cp -f $SC4S_ETC/conf.d/configmap/context/splunk_metadata.csv $SC4S_ETC/conf.d/local/context/splunk_metadata.csv - #cp -R -f $SC4S_ETC/conf.d/configmap/config/* $SC4S_ETC/conf.d/local/config/app_parsers/ if [[ -f $SC4S_ETC/conf.d/configmap/addons/config.yaml ]]; then cp $SC4S_ETC/conf.d/configmap/addons/config.yaml $SC4S_ETC/config.yaml fi @@ -101,10 +129,12 @@ else cp -R -f $SC4S_ETC/local_config/* $SC4S_ETC/conf.d/local/config/ fi +# Generate main config file for syslog engine from jinja2 template if [[ -f $SC4S_ETC/syslog-ng.conf.jinja ]]; then python3 -m config_generator --config=$SC4S_ETC/config.yaml > $SC4S_ETC/syslog-ng.conf fi +# Adds examples of different parsers to sc4s local dirctory if [ "$TEST_SC4S_ACTIVATE_EXAMPLES" == "yes" ] then for file in $SC4S_ETC/conf.d/local/context/*.example ; do cp --verbose -n $file ${file%.example}; done @@ -113,6 +143,7 @@ fi for file in $SC4S_ETC/conf.d/local/context/*.example ; do touch ${file%.example}; done touch $SC4S_ETC/conf.d/local/context/splunk_metadata.csv +# Generating and storing TLS Certificate if [ "$SC4S_SOURCE_TLS_SELFSIGNED" == "yes" ] then mkdir -p $SC4S_TLS || true @@ -123,18 +154,8 @@ then openssl x509 -req -in ${SC4S_TLS}/server.csr -CA ${SC4S_TLS}/ca.crt -CAkey ${SC4S_TLS}/ca.key -CAcreateserial -out ${SC4S_TLS}/server.pem fi fi -# if [ -f "${SC4S_TLS}/trusted.pem" ] -# then -# cp ${SC4S_TLS}/trusted.pem /usr/share/pki/ca-trust-source/anchors/ -# update-ca-trust -# fi -# if [ -f "${SC4S_TLS}/ca.crt" ] -# then -# cp ${SC4S_TLS}/trusted.pem /usr/share/pki/ca-trust-source/anchors/ -# update-ca-trust -# fi - -# Check Linux distribution if its alpine + +# Check Linux distribution and store TLS certs if grep -q 'alpine' /etc/os-release; then IS_ALPINE=true else @@ -164,14 +185,14 @@ else update-ca-trust fi fi -# Test HEC Connectivity + +# Set HEC indexes and test connectivity with sending "HEC TEST EVENT" SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=$(echo $SC4S_DEST_SPLUNK_HEC_DEFAULT_URL | sed 's/\(https\{0,1\}\:\/\/[^\/, ]*\)[^, ]*/\1\/services\/collector\/event/g' | sed 's/,/ /g') if [ "$SC4S_DEST_SPLUNK_HEC_GLOBAL" != "no" ] then HEC=$(echo $SC4S_DEST_SPLUNK_HEC_DEFAULT_URL | cut -d' ' -f 1) if [ "${SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY}" == "no" ]; then export NO_VERIFY=-k ; fi - - export SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_MOUNT=${SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_MOUNT:=${SC4S_DEST_TLS_MOUNT}} + if [ -n "${SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_MOUNT}" ]; then export HEC_TLS_OPTS="--cert ${SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_MOUNT}/cert.pem --key ${SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_MOUNT}/key.pem --cacert ${SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_MOUNT}/ca_cert.pem"; else @@ -199,18 +220,22 @@ then fi fi -if [ "${SC4S_CLEAR_NAME_CACHE}" == "yes" ] || [ "${SC4S_CLEAR_NAME_CACHE}" == "1" ] +# Clearing the local db that stores ip host pairs +if [ "${SC4S_CLEAR_NAME_CACHE}" == "yes" ] || [ "${SC4S_CLEAR_NAME_CACHE}" == "1" ] || [ "${SC4S_CLEAR_NAME_CACHE}" == "true" ] then rm -f $SC4S_VAR/hostip.sqlite echo "hostip.sqlite file deleted at $SC4S_VAR" fi -# Create a workable variable with a list of simple log paths +# Create a workable variable with a list of simple log paths, used in port validation script export SOURCE_SIMPLE_SET=$(printenv | grep '^SC4S_LISTEN_SIMPLE_.*_PORT=.' | sed 's/^SC4S_LISTEN_SIMPLE_//;s/_..._PORT\=.*//;s/_[^_]*_PORT\=.*//' | sort | uniq | xargs echo | sed 's/ /,/g' | tr '[:upper:]' '[:lower:]' ) export SOURCE_ALL_SET=$(printenv | grep '^SC4S_LISTEN_.*_PORT=.' | grep -v "disabled" | sed 's/^SC4S_LISTEN_//;s/_..._PORT\=.*//;s/_[^_]*_PORT\=.*//' | sort | uniq | xargs echo | sed 's/ /,/g' | tr '[:lower:]' '[:upper:]' ) +# Validate ports python3 /source_ports_validator.py + +# Generate csv with vendor to Splunk index mappings, to be filled with correct index later syslog-ng --no-caps --preprocess-into=- | grep vendor_product | grep set | grep -v 'set(.\$' | sed 's/^ *//' | grep 'value("fields.sc4s_vendor_product"' | grep -v "\`vendor_product\`" | sed s/^set\(// | cut -d',' -f1 | sed 's/\"//g' >/tmp/keys syslog-ng --no-caps --preprocess-into=- | grep 'meta_key(.' | sed 's/^ *meta_key(.//' | sed "s/')//" >>/tmp/keys rm -f $SC4S_ETC/conf.d/local/context/splunk_metadata.csv.example >/dev/null || true @@ -218,6 +243,7 @@ for fn in `cat /tmp/keys | sort | uniq`; do echo "${fn},index,setme" >>$SC4S_ETC/conf.d/local/context/splunk_metadata.csv.example done +# Checking configuration and running a healthcheck echo syslog-ng checking config export SC4S_VERSION=$(cat $SC4S_ETC/VERSION) echo sc4s version=$(cat $SC4S_ETC/VERSION) @@ -227,6 +253,7 @@ echo sc4s version=$(cat $SC4S_ETC/VERSION) >>$SC4S_VAR/log/syslog-ng.out echo "Configuring the health check port to: $SC4S_LISTEN_STATUS_PORT" nohup gunicorn -b 0.0.0.0:$SC4S_LISTEN_STATUS_PORT healthcheck:app & +# Generating syslog configuration and export it to tmp file # OPTIONAL for BYOE: Comment out/remove all remaining lines and launch syslog-ng directly from systemd if [ "${SC4S_DEBUG_CONTAINER}" == "yes" ] then @@ -235,6 +262,7 @@ then export >/tmp/export_file fi +# Check syntax of syslog configuration syslog-ng -s --no-caps if [ $? != 0 ] then @@ -246,6 +274,7 @@ then fi fi +# Loop that runs and restarts syslog-ng, reacts to specific signals (exit codes - 147) to exit syslog-ng while : do echo starting syslog-ng @@ -261,12 +290,15 @@ do then echo "syslog-ng failed to start; exiting..." fi + # Wait returns exit status of process, exit status = 128 + process_id wait ${pid} - if [ $? == 147 ] + exit_code=$? + # 147 - SIGSTOP(19), 143 - SIGTERM(15), 134 - SIGABRT(6), 130 - SIGINT(2) + if [ $exit_code == 147 ] || [ $exit_code == 143 ] || [ $exit_code == 134 ] || [ $exit_code == 130 ] then - exit $? + exit $exit_code else - echo "Handling exit $? and restarting" + echo "Handling exit $exit_code and restarting" fi fi done diff --git a/poetry.lock b/poetry.lock index b7957dbda4..a938c7e31f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -512,23 +512,24 @@ files = [ [[package]] name = "flask" -version = "3.1.0" +version = "3.1.1" description = "A simple framework for building complex web applications." optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "flask-3.1.0-py3-none-any.whl", hash = "sha256:d667207822eb83f1c4b50949b1623c8fc8d51f2341d65f72e1a1815397551136"}, - {file = "flask-3.1.0.tar.gz", hash = "sha256:5f873c5184c897c8d9d1b05df1e3d01b14910ce69607a117bd3277098a5836ac"}, + {file = "flask-3.1.1-py3-none-any.whl", hash = "sha256:07aae2bb5eaf77993ef57e357491839f5fd9f4dc281593a81a9e4d79a24f295c"}, + {file = "flask-3.1.1.tar.gz", hash = "sha256:284c7b8f2f58cb737f0cf1c30fd7eaf0ccfcde196099d24ecede3fc2005aa59e"}, ] [package.dependencies] -blinker = ">=1.9" +blinker = ">=1.9.0" click = ">=8.1.3" -importlib-metadata = {version = ">=3.6", markers = "python_version < \"3.10\""} -itsdangerous = ">=2.2" -Jinja2 = ">=3.1.2" -Werkzeug = ">=3.1" +importlib-metadata = {version = ">=3.6.0", markers = "python_version < \"3.10\""} +itsdangerous = ">=2.2.0" +jinja2 = ">=3.1.2" +markupsafe = ">=2.1.1" +werkzeug = ">=3.1.0" [package.extras] async = ["asgiref (>=3.2)"] @@ -1507,23 +1508,24 @@ files = [ [[package]] name = "tornado" -version = "6.4.2" +version = "6.5" description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["main"] files = [ - {file = "tornado-6.4.2-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:e828cce1123e9e44ae2a50a9de3055497ab1d0aeb440c5ac23064d9e44880da1"}, - {file = "tornado-6.4.2-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:072ce12ada169c5b00b7d92a99ba089447ccc993ea2143c9ede887e0937aa803"}, - {file = "tornado-6.4.2-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a017d239bd1bb0919f72af256a970624241f070496635784d9bf0db640d3fec"}, - {file = "tornado-6.4.2-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c36e62ce8f63409301537222faffcef7dfc5284f27eec227389f2ad11b09d946"}, - {file = "tornado-6.4.2-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bca9eb02196e789c9cb5c3c7c0f04fb447dc2adffd95265b2c7223a8a615ccbf"}, - {file = "tornado-6.4.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:304463bd0772442ff4d0f5149c6f1c2135a1fae045adf070821c6cdc76980634"}, - {file = "tornado-6.4.2-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:c82c46813ba483a385ab2a99caeaedf92585a1f90defb5693351fa7e4ea0bf73"}, - {file = "tornado-6.4.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:932d195ca9015956fa502c6b56af9eb06106140d844a335590c1ec7f5277d10c"}, - {file = "tornado-6.4.2-cp38-abi3-win32.whl", hash = "sha256:2876cef82e6c5978fde1e0d5b1f919d756968d5b4282418f3146b79b58556482"}, - {file = "tornado-6.4.2-cp38-abi3-win_amd64.whl", hash = "sha256:908b71bf3ff37d81073356a5fadcc660eb10c1476ee6e2725588626ce7e5ca38"}, - {file = "tornado-6.4.2.tar.gz", hash = "sha256:92bad5b4746e9879fd7bf1eb21dce4e3fc5128d71601f80005afa39237ad620b"}, + {file = "tornado-6.5-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:f81067dad2e4443b015368b24e802d0083fecada4f0a4572fdb72fc06e54a9a6"}, + {file = "tornado-6.5-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9ac1cbe1db860b3cbb251e795c701c41d343f06a96049d6274e7c77559117e41"}, + {file = "tornado-6.5-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c625b9d03f1fb4d64149c47d0135227f0434ebb803e2008040eb92906b0105a"}, + {file = "tornado-6.5-cp39-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9a0d8d2309faf015903080fb5bdd969ecf9aa5ff893290845cf3fd5b2dd101bc"}, + {file = "tornado-6.5-cp39-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:03576ab51e9b1677e4cdaae620d6700d9823568b7939277e4690fe4085886c55"}, + {file = "tornado-6.5-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ab75fe43d0e1b3a5e3ceddb2a611cb40090dd116a84fc216a07a298d9e000471"}, + {file = "tornado-6.5-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:119c03f440a832128820e87add8a175d211b7f36e7ee161c631780877c28f4fb"}, + {file = "tornado-6.5-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:231f2193bb4c28db2bdee9e57bc6ca0cd491f345cd307c57d79613b058e807e0"}, + {file = "tornado-6.5-cp39-abi3-win32.whl", hash = "sha256:fd20c816e31be1bbff1f7681f970bbbd0bb241c364220140228ba24242bcdc59"}, + {file = "tornado-6.5-cp39-abi3-win_amd64.whl", hash = "sha256:007f036f7b661e899bd9ef3fa5f87eb2cb4d1b2e7d67368e778e140a2f101a7a"}, + {file = "tornado-6.5-cp39-abi3-win_arm64.whl", hash = "sha256:542e380658dcec911215c4820654662810c06ad872eefe10def6a5e9b20e9633"}, + {file = "tornado-6.5.tar.gz", hash = "sha256:c70c0a26d5b2d85440e4debd14a8d0b463a0cf35d92d3af05f5f1ffa8675c826"}, ] [[package]] @@ -1540,14 +1542,14 @@ files = [ [[package]] name = "typing-extensions" -version = "4.13.1" -description = "Backported and Experimental Type Hints for Python 3.8+" +version = "4.14.0" +description = "Backported and Experimental Type Hints for Python 3.9+" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "typing_extensions-4.13.1-py3-none-any.whl", hash = "sha256:4b6cf02909eb5495cfbc3f6e8fd49217e6cc7944e145cdda8caa3734777f9e69"}, - {file = "typing_extensions-4.13.1.tar.gz", hash = "sha256:98795af00fb9640edec5b8e31fc647597b4691f099ad75f469a2616be1a76dff"}, + {file = "typing_extensions-4.14.0-py3-none-any.whl", hash = "sha256:a1514509136dd0b477638fc68d6a91497af5076466ad0fa6c338e44e359944af"}, + {file = "typing_extensions-4.14.0.tar.gz", hash = "sha256:8676b788e32f02ab42d9e7c61324048ae4c6d844a399eebace3d4979d75ceef4"}, ] [[package]] diff --git a/tests/test_citrix_netscaler.py b/tests/test_citrix_netscaler.py index b7e6413298..ef6f661c6d 100644 --- a/tests/test_citrix_netscaler.py +++ b/tests/test_citrix_netscaler.py @@ -4,6 +4,9 @@ # license that can be found in the LICENSE-BSD2 file or at # https://opensource.org/licenses/BSD-2-Clause import datetime +import os +from unittest.mock import patch + import shortuuid import pytz import pytest @@ -28,7 +31,7 @@ def test_citrix_netscaler(record_property, setup_splunk, setup_sc4s, get_pid): _, bsd, time, _, _, tzname, epoch = time_operations(dt) # Tune time functions - time = dt.strftime("%d/%m/%Y:%H:%M:%S") + time = dt.strftime("%m/%d/%Y:%H:%M:%S") epoch = epoch[:-7] mt = env.from_string( @@ -91,6 +94,49 @@ def test_citrix_netscaler_sdx( assert result_count == 1 +# <134> 05/08/2025:03:13:15 GMT DC-NS02 0-PPE-0 : default TCP CONN_TERMINATE 1874124822 0 : Source 10.x.x.x:47990 - Destination 10.x.x.x:80 - Start Time 26/03/2025:21:13:15 GMT - End Time 26/03/2025:21:13:15 GMT - Total_bytes_send 1 - Total_bytes_recv 1 +@pytest.mark.addons("citrix") +@patch.dict( + os.environ, + { + "SC4S_IGNORE_MMDD_LEGACY_CITRIX_NETSCALER": "yes", + }, + clear=False +) +def test_citrix_netscaler_new_date_format( + record_property, setup_splunk, setup_sc4s, get_pid +): + host = f"test-ctitrixns-host-{shortuuid.ShortUUID().random(length=5).lower()}-{shortuuid.ShortUUID().random(length=5).lower()}" + pid = get_pid + + dt = datetime.datetime.now(datetime.timezone.utc) + _, bsd, time, _, _, tzname, epoch = time_operations(dt) + + # Tune time functions + time = dt.strftime("%d/%m/%Y:%H:%M:%S") + epoch = epoch[:-7] + + mt = env.from_string( + "{{ mark }} {{ time }} GMT {{ host }} 0-PPE-0 : default TCP CONN_TERMINATE 1874124822 0 : Source 10.x.x.x:47990 - Destination 10.x.x.x:80 - Start Time 26/03/2025:21:13:15 GMT - End Time 26/03/2025:21:13:15 GMT - Total_bytes_send 1 - Total_bytes_recv 1\n" + ) + message = mt.render( + mark="<134>", bsd=bsd, time=time, tzname=tzname, host=host, pid=pid + ) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} index=netfw host={{ host }} sourcetype="citrix:netscaler:syslog"' + ) + search = st.render(epoch=epoch, host=host, pid=pid) + + result_count, _ = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", result_count) + record_property("message", message) + + assert result_count == 1 # [289]: AAA Message : In receive_ldap_user_search_event: ldap_first_entry returned null, user ssgconfig not found @pytest.mark.addons("citrix") diff --git a/tests/test_dell_powerstore.py b/tests/test_dell_powerstore.py new file mode 100644 index 0000000000..4b81b7110e --- /dev/null +++ b/tests/test_dell_powerstore.py @@ -0,0 +1,65 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause + +from jinja2 import Environment, select_autoescape + +from .sendmessage import sendsingle +from .splunkutils import splunk_single +from .timeutils import time_operations +import datetime + +import pytest + +env = Environment(autoescape=select_autoescape(default_for_string=False)) + +# <110>Jan 31 19:43:24 APM00243620939-B [358]: 2025-01-31T19:43:17 APM00243620939-B PSb8ad27c26647 358@HM3CTZ3 Authentication [PowerStore_audit_event@1139 id="2341" user="admin" resource_type="login_session" action="None" client_ip="10.114.173.252" appliance="APM00243620939" status="success"] User "admin" logged in successfully. +# <110>Jan 31 19:44:44 APM00243620939-B [358]: 2025-01-31T19:44:31 APM00243620939-B PSb8ad27c26647 358@HM3CTZ3 Authentication [PowerStore_audit_event@1139 id="2342" user="EncryptHTTP.PSb8ad27c26647" resource_type="login_session" action="None" client_ip="None" appliance="APM00243620939" status="success"] Successfully authenticated cert_account : Dell EMC PowerStore CA P9XEU8F5/EncryptHTTP.PSb8ad27c26647. +# <110>Jan 31 19:45:44 APM00243620939-B [358]: 2025-01-31T19:45:29 APM00243620939-B PSb8ad27c26647 358@HM3CTZ3 Service [PowerStore_audit_event@1139 id="2347" user="root" resource_type="unknown" action="not applicable" client_ip="not applicable" appliance="APM00243620939" status="success"] User root executed the service script command [/cyc_host/cyc_service/bin/svc_diag list --hardware --sub_options firmware] from APM00243620939-A via shell. +# <110>Jan 31 19:48:25 APM00243620939-B [358]: 2025-01-31T19:48:16 APM00243620939-B PSb8ad27c26647 358@HM3CTZ3 Authentication [PowerStore_audit_event@1139 id="2349" user="EncryptHTTP.PSb8ad27c26647" resource_type="login_session" action="None" client_ip="None" appliance="APM00243620939" status="success"] Successfully authenticated cert_account : Dell EMC PowerStore CA P9XEU8F5/EncryptHTTP.PSb8ad27c26647. +# <110>Jan 31 19:49:05 APM00243620939-B [358]: 2025-01-31T19:48:49 APM00243620939-B PSb8ad27c26647 358@HM3CTZ3 Config [PowerStore_audit_event@1139 id="2351" user="admin" resource_type="system_health_check" action="create" client_ip="10.114.173.252" appliance="APM00243620939" status="failed"] Failed to perform system health check on pki-tech-ps-p01. +# <110>Jan 31 19:58:46 APM00243620939-B [358]: 2025-01-31T19:58:22 APM00243620939-B PSb8ad27c26647 358@HM3CTZ3 Logout [PowerStore_audit_event@1139 id="2352" user="admin" resource_type="login_session" action="delete" client_ip="10.114.173.252" appliance="APM00243620939" status="success"] User "admin" was successfully logged out. + +test_cases = [ + "{{ mark }}{{ bsd }} {{ host }} [358]: {{ iso }} {{ host }} PSb8ad27c26647 358@HM3CTZ3 Authentication [PowerStore_audit_event@1139 id=\"2341\" user=\"admin\" resource_type=\"login_session\" action=\"None\" client_ip=\"10.114.173.252\" appliance=\"APM00243620939\" status=\"success\"] User \"admin\" logged in successfully.", + "{{ mark }}{{ bsd }} {{ host }} [358]: {{ iso }} {{ host }} PSb8ad27c26647 358@HM3CTZ3 Authentication [PowerStore_audit_event@1139 id=\"2342\" user=\"EncryptHTTP.PSb8ad27c26647\" resource_type=\"login_session\" action=\"None\" client_ip=\"None\" appliance=\"APM00243620939\" status=\"success\"] Successfully authenticated cert_account : Dell EMC PowerStore CA P9XEU8F5/EncryptHTTP.PSb8ad27c26647.", + "{{ mark }}{{ bsd }} {{ host }} [358]: {{ iso }} {{ host }} PSb8ad27c26647 358@HM3CTZ3 Service [PowerStore_audit_event@1139 id=\"2347\" user=\"root\" resource_type=\"unknown\" action=\"not applicable\" client_ip=\"not applicable\" appliance=\"APM00243620939\" status=\"success\"] User root executed the service script command [/cyc_host/cyc_service/bin/svc_diag list --hardware --sub_options firmware] from APM00243620939-A via shell.", + "{{ mark }}{{ bsd }} {{ host }} [358]: {{ iso }} {{ host }} PSb8ad27c26647 358@HM3CTZ3 Authentication [PowerStore_audit_event@1139 id=\"2349\" user=\"EncryptHTTP.PSb8ad27c26647\" resource_type=\"login_session\" action=\"None\" client_ip=\"None\" appliance=\"APM00243620939\" status=\"success\"] Successfully authenticated cert_account : Dell EMC PowerStore CA P9XEU8F5/EncryptHTTP.PSb8ad27c26647.", + "{{ mark }}{{ bsd }} {{ host }} [358]: {{ iso }} {{ host }} PSb8ad27c26647 358@HM3CTZ3 Config [PowerStore_audit_event@1139 id=\"2351\" user=\"admin\" resource_type=\"system_health_check\" action=\"create\" client_ip=\"10.114.173.252\" appliance=\"APM00243620939\" status=\"failed\"] Failed to perform system health check on pki-tech-ps-p01.", + "{{ mark }}{{ bsd }} {{ host }} [358]: {{ iso }} {{ host }} PSb8ad27c26647 358@HM3CTZ3 Logout [PowerStore_audit_event@1139 id=\"2352\" user=\"admin\" resource_type=\"login_session\" action=\"delete\" client_ip=\"10.114.173.252\" appliance=\"APM00243620939\" status=\"success\"] User \"admin\" was successfully logged out." + +] + + +@pytest.mark.parametrize("case", test_cases) +@pytest.mark.addons("dell") +def test_dell_powerstore( + record_property, setup_splunk, setup_sc4s, case +): + host = f'test-dell-powerstore-{test_cases.index(case)}' + + dt = datetime.datetime.now() + iso, bsd, _, _, _, _, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string(case + "\n") + message = mt.render(mark="<110>", bsd=bsd, host=host, iso=iso) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search index=netops _time={{ epoch }} sourcetype="dell:emc:powerstore" (host="{{ host }}" OR "{{ host }}")' + ) + search = st.render(epoch=epoch, host=host) + + result_count, _ = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", result_count) + record_property("message", message) + + assert result_count == 1