feat: update entrypoint script

docs/upgrade.md

@@ -18,6 +18,10 @@ For a step by step guide [see here](./v3_upgrade.md).
 
 You may need to migrate legacy log paths or version 1 app-parsers for version 2. To do this, open an issue and attach the original configuration and a compressed pcap of sample data for testing. We will evaluate whether to include the source in an upcoming release.
 
+### Upgrade from <3.37.0
+In `entrypoint.sh` the old variable mappings `SPLUNK_HEC_URL`, `SPLUNK_HEC_TOKEN`, `SC4S_DEST_SPLUNK_HEC_TLS_VERIFY` are deprecated and will not be 
+further reassigned, instead use `SC4S_DEST_SPLUNK_HEC_DEFAULT_URL`, `SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN`, `SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY`.
+
 ### Upgrade from <3.33.0
 In NetApp ONTAP, the ontap:ems sourcetype has been updated to netapp:ontap:audit, so old logs are now classified under netapp:ontap:audit. Additionally, a new netapp:ontap:ems sourcetype has been introduced. If you upgrade and want these new changes, ensure that you set `SC4S_NETAPP_ONTAP_NEW_FORMAT` environment variable to `yes` and configure your system to send the logs to a specific port or have a hostname-based configuration in place for proper log onboarding into Splunk.
 

package/sbin/entrypoint.sh

@@ -1,13 +1,15 @@
 #!/usr/bin/env bash
 function join_by { local d=$1; shift; local f=$1; shift; printf %s "$f" "${@/#/$d}"; }
+
+# Activate python environment and run parsing/caching for conf files
 . /var/lib/python-venv/bin/activate
 export PYTHONPATH=/etc/syslog-ng/pylib
-
 python3 /etc/syslog-ng/pylib/parser_source_cache.py
 
+# Configuring environment variables
 export SC4S_LISTEN_STATUS_PORT=${SC4S_LISTEN_STATUS_PORT:=8080}
 
-# These path variables allow for a single entrypoint script to be utilized for both Container and BYOE runtimes
+
 export SC4S_LISTEN_DEFAULT_TCP_PORT=${SC4S_LISTEN_DEFAULT_TCP_PORT:=514}
 export SC4S_LISTEN_DEFAULT_UDP_PORT=${SC4S_LISTEN_DEFAULT_UDP_PORT:=514}
 export SC4S_LISTEN_DEFAULT_TLS_PORT=${SC4S_LISTEN_DEFAULT_TLS_PORT:=6514}
@@ -22,20 +24,19 @@ export SC4S_DEST_SPLUNK_INDEXED_FIELDS=${SC4S_DEST_SPLUNK_INDEXED_FIELDS:=r_unix
 
 export SC4S_OPTION_FORTINET_SOURCETYPE_PREFIX=${SC4S_OPTION_FORTINET_SOURCETYPE_PREFIX:=fgt}
 
-if [ -n "${SPLUNK_HEC_URL}" ]; then export SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=$SPLUNK_HEC_URL; fi
-if [ -n "${SPLUNK_HEC_TOKEN}" ]; then export SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=$SPLUNK_HEC_TOKEN; fi
-if [ -n "${SC4S_DEST_SPLUNK_HEC_TLS_VERIFY}" ]; then export SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY=$SC4S_DEST_SPLUNK_HEC_TLS_VERIFY; fi
-
+# Variables with path to sc4s directories
+# These path variables allow for a single entrypoint script to be utilized for both Container and BYOE runtimes
 export SC4S_ETC=${SC4S_ETC:=/etc/syslog-ng}
 export SC4S_TLS=${SC4S_TLS:=/etc/syslog-ng/tls}
 export SC4S_VAR=${SC4S_VAR:=/var/lib/syslog-ng}
 export SC4S_BIN=${SC4S_BIN:=/usr/bin}
 export SC4S_SBIN=${SC4S_SBIN:=/usr/sbin}
 
+# Set list with alternate destinations than HEC
 export SC4S_DESTS_FILTERED_ALTERNATES=$(env | grep _FILTERED_ALTERNATES= | grep -v SC4S_DEST_GLOBAL_FILTERED_ALTERNATES | cut -d= -f2 | sort | uniq |  paste -s -d, -)
 [ -z "$SC4S_DESTS_FILTERED_ALTERNATES" ] && unset SC4S_DESTS_FILTERED_ALTERNATES
 
-# SIGTERM-handler
+# SIGTERM(15) - requests termination (default signal for kill)
 term_handler() {
 # SIGTERM on valid PID; return exit code 0 (clean exit)
   if [ $pid -ne 0 ]; then
@@ -48,15 +49,15 @@ term_handler() {
   exit 143
 }
 
-# SIGHUP-handler
+# SIGHUP(1) - used to reload configs or restart processes
 hup_handler() {
   if [ $pid -ne 0 ]; then
     echo Reloading syslog-ng...
     kill -SIGHUP ${pid}
   fi
 }
 
-# SIGQUIT-handler
+# SIGQUIT(3) - used on process to quit and dump core
 quit_handler() {
   if [ $pid -ne 0 ]; then
     echo Quitting syslog-ng...
@@ -65,10 +66,39 @@ quit_handler() {
   fi
 }
 
+# SIGABRT(6) - abort signal
+abrt_handler() {
+# SIGABRT on valid PID
+  if [ $pid -ne 0 ]; then
+    echo Aborting syslog-ng...
+    kill -SIGABRT ${pid}
+    wait ${pid}
+    exit $?
+  fi
+# 128 + 6
+  exit 134
+}
+
+# SIGINT(2) - interrupts the process (ex. Ctrl+C)
+int_handler() {
+  if [ $pid -ne 0 ]; then
+    echo Interupting syslog-ng...
+    kill -SIGINT ${pid}
+    wait ${pid}
+    exit $?
+  fi
+# 128 + 2
+  exit 130
+}
+
+# Setting traps to run handler function based on received signal
 trap 'kill ${!}; hup_handler' SIGHUP
 trap 'kill ${!}; term_handler' SIGTERM
 trap 'kill ${!}; quit_handler' SIGQUIT
+trap 'kill ${!}; abrt_handler' SIGABRT
+trap 'kill ${!}; int_handler' SIGINT
 
+# Create directories needed for SC4S
 mkdir -p $SC4S_VAR/log/
 mkdir -p $SC4S_ETC/conf.d/local/context/
 mkdir -p $SC4S_ETC/conf.d/merged/context/
@@ -80,6 +110,7 @@ mkdir -p $SC4S_ETC/addons/
 # copy all files in context_templates to conf.d/local/context
 cp -f $SC4S_ETC/context_templates/* $SC4S_ETC/conf.d/local/context
 
+# Copying the config files from sc4s repository to sc4s local directory
 # check if runtime environment is k8s
 if [ "$SC4S_RUNTIME_ENV" == "k8s" ]
 then
@@ -89,10 +120,7 @@ then
   mkdir -p $SC4S_ETC/conf.d/configmap/addons/
 
   # copy all files in configmap/context to conf.d/local/context
-
   cp -R -f $SC4S_ETC/conf.d/configmap/* $SC4S_ETC/conf.d/local/
-  #cp -f $SC4S_ETC/conf.d/configmap/context/splunk_metadata.csv $SC4S_ETC/conf.d/local/context/splunk_metadata.csv
-  #cp -R -f $SC4S_ETC/conf.d/configmap/config/* $SC4S_ETC/conf.d/local/config/app_parsers/
   if [[ -f $SC4S_ETC/conf.d/configmap/addons/config.yaml ]]; then
     cp $SC4S_ETC/conf.d/configmap/addons/config.yaml $SC4S_ETC/config.yaml
   fi
@@ -101,10 +129,12 @@ else
   cp -R -f $SC4S_ETC/local_config/* $SC4S_ETC/conf.d/local/config/
 fi
 
+# Generate main config file for syslog engine from jinja2 template
 if [[ -f $SC4S_ETC/syslog-ng.conf.jinja ]]; then
   python3 -m config_generator --config=$SC4S_ETC/config.yaml > $SC4S_ETC/syslog-ng.conf
 fi
 
+# Adds examples of different parsers to sc4s local dirctory
 if [ "$TEST_SC4S_ACTIVATE_EXAMPLES" == "yes" ]
 then
   for file in $SC4S_ETC/conf.d/local/context/*.example ; do cp --verbose -n $file ${file%.example}; done
@@ -113,6 +143,7 @@ fi
 for file in $SC4S_ETC/conf.d/local/context/*.example ; do touch ${file%.example}; done
 touch $SC4S_ETC/conf.d/local/context/splunk_metadata.csv
 
+# Generating and storing TLS Certificate
 if [ "$SC4S_SOURCE_TLS_SELFSIGNED" == "yes" ]
 then
   mkdir -p $SC4S_TLS || true
@@ -123,18 +154,8 @@ then
     openssl x509 -req -in ${SC4S_TLS}/server.csr -CA ${SC4S_TLS}/ca.crt -CAkey ${SC4S_TLS}/ca.key -CAcreateserial -out ${SC4S_TLS}/server.pem
   fi
 fi
-# if [ -f "${SC4S_TLS}/trusted.pem" ]
-# then
-#   cp ${SC4S_TLS}/trusted.pem /usr/share/pki/ca-trust-source/anchors/
-#   update-ca-trust
-# fi
-# if [ -f "${SC4S_TLS}/ca.crt" ]
-# then
-#   cp ${SC4S_TLS}/trusted.pem /usr/share/pki/ca-trust-source/anchors/
-#   update-ca-trust
-# fi 
-
-# Check Linux distribution if its alpine
+
+# Check Linux distribution and store TLS certs
 if grep -q 'alpine' /etc/os-release; then
   IS_ALPINE=true
 else
@@ -164,14 +185,14 @@ else
     update-ca-trust
   fi
 fi
-# Test HEC Connectivity
+
+# Set HEC indexes and test connectivity with sending "HEC TEST EVENT"
 SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=$(echo $SC4S_DEST_SPLUNK_HEC_DEFAULT_URL | sed 's/\(https\{0,1\}\:\/\/[^\/, ]*\)[^, ]*/\1\/services\/collector\/event/g' | sed 's/,/ /g')
 if [ "$SC4S_DEST_SPLUNK_HEC_GLOBAL" != "no" ]
 then
   HEC=$(echo $SC4S_DEST_SPLUNK_HEC_DEFAULT_URL | cut -d' ' -f 1)
   if [ "${SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY}" == "no" ]; then export NO_VERIFY=-k ; fi
-
-  export SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_MOUNT=${SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_MOUNT:=${SC4S_DEST_TLS_MOUNT}}
+
   if [ -n "${SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_MOUNT}" ]; then
     export HEC_TLS_OPTS="--cert ${SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_MOUNT}/cert.pem  --key ${SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_MOUNT}/key.pem --cacert ${SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_MOUNT}/ca_cert.pem";
   else
@@ -199,25 +220,30 @@ then
   fi
 fi
 
-if [ "${SC4S_CLEAR_NAME_CACHE}" == "yes" ] || [ "${SC4S_CLEAR_NAME_CACHE}" == "1" ]
+# Clearing the local db that stores ip host pairs
+if [ "${SC4S_CLEAR_NAME_CACHE}" == "yes" ] || [ "${SC4S_CLEAR_NAME_CACHE}" == "1" ] || [ "${SC4S_CLEAR_NAME_CACHE}" == "true" ]
 then 
   rm -f $SC4S_VAR/hostip.sqlite
   echo "hostip.sqlite file deleted at $SC4S_VAR"
 fi
 
-# Create a workable variable with a list of simple log paths
+# Create a workable variable with a list of simple log paths, used in port validation script
 export SOURCE_SIMPLE_SET=$(printenv | grep '^SC4S_LISTEN_SIMPLE_.*_PORT=.' | sed 's/^SC4S_LISTEN_SIMPLE_//;s/_..._PORT\=.*//;s/_[^_]*_PORT\=.*//' | sort | uniq |  xargs echo | sed 's/ /,/g' | tr '[:upper:]' '[:lower:]' )
 export SOURCE_ALL_SET=$(printenv | grep '^SC4S_LISTEN_.*_PORT=.' | grep -v "disabled" | sed 's/^SC4S_LISTEN_//;s/_..._PORT\=.*//;s/_[^_]*_PORT\=.*//' | sort | uniq |  xargs echo | sed 's/ /,/g' | tr '[:lower:]' '[:upper:]' )
 
+# Validate ports
 python3 /source_ports_validator.py
 
+
+# Generate csv with vendor to Splunk index mappings, to be filled with correct index later
 syslog-ng --no-caps --preprocess-into=- | grep vendor_product | grep set | grep -v 'set(.\$' | sed 's/^ *//' | grep 'value("fields.sc4s_vendor_product"' | grep -v "\`vendor_product\`" | sed s/^set\(// | cut -d',' -f1 | sed 's/\"//g' >/tmp/keys
 syslog-ng --no-caps --preprocess-into=- | grep 'meta_key(.' | sed 's/^ *meta_key(.//' | sed "s/')//" >>/tmp/keys
 rm -f $SC4S_ETC/conf.d/local/context/splunk_metadata.csv.example >/dev/null || true
 for fn in `cat /tmp/keys | sort | uniq`; do
     echo "${fn},index,setme" >>$SC4S_ETC/conf.d/local/context/splunk_metadata.csv.example
 done
 
+# Checking configuration and running a healthcheck
 echo syslog-ng checking config
 export SC4S_VERSION=$(cat $SC4S_ETC/VERSION)
 echo sc4s version=$(cat $SC4S_ETC/VERSION)
@@ -227,6 +253,7 @@ echo sc4s version=$(cat $SC4S_ETC/VERSION) >>$SC4S_VAR/log/syslog-ng.out
 echo "Configuring the health check port to: $SC4S_LISTEN_STATUS_PORT"
 nohup gunicorn -b 0.0.0.0:$SC4S_LISTEN_STATUS_PORT healthcheck:app &
 
+# Generating syslog configuration and export it to tmp file
 # OPTIONAL for BYOE:  Comment out/remove all remaining lines and launch syslog-ng directly from systemd
 if [ "${SC4S_DEBUG_CONTAINER}" == "yes" ]
 then
@@ -235,6 +262,7 @@ then
   export >/tmp/export_file
 fi
 
+# Check syntax of syslog configuration
 syslog-ng -s --no-caps
 if [ $? != 0 ]
 then
@@ -246,6 +274,7 @@ then
   fi
 fi
 
+# Loop that runs and restarts syslog-ng, reacts to specific signals (exit codes - 147) to exit syslog-ng
 while :
 do
   echo starting syslog-ng
@@ -261,12 +290,15 @@ do
     then
       echo "syslog-ng failed to start; exiting..."
     fi
+    # Wait returns exit status of process, exit status = 128 + process_id
     wait ${pid}
-    if [ $? == 147 ]
+    exit_code=$?
+    # 147 - SIGSTOP(19), 143 - SIGTERM(15), 134 - SIGABRT(6), 130 - SIGINT(2)
+    if [ $exit_code == 147  ] || [ $exit_code == 143 ] || [ $exit_code == 134 ] || [ $exit_code == 130 ]
     then
-      exit $?
+      exit $exit_code
     else
-      echo "Handling exit $? and restarting"
+      echo "Handling exit $exit_code and restarting"
     fi
   fi
 done