update: handle graceful shutdown in Node container (#2337)

Signed-off-by: Viet Nguyen Duc <[email protected]>

Author: VietND96

.gitignore

@@ -24,6 +24,7 @@ StandaloneC*/generate_config
 StandaloneF*/generate_config
 StandaloneE*/generate_config
 videos
+Base/configs
 
 # Created by https://www.gitignore.io/api/virtualenv
 

Base/Dockerfile

@@ -127,7 +127,7 @@ RUN  if [ "$(dpkg --print-architecture)" = "amd64" ]; then \
 #======================================
 # Add Grid check script
 #======================================
-COPY --chown="${SEL_UID}:${SEL_GID}" check-grid.sh entry_point.sh /opt/bin/
+COPY --chown="${SEL_UID}:${SEL_GID}" check-grid.sh entry_point.sh configs/node/nodeGridUrl.sh configs/node/nodePreStop.sh /opt/bin/
 
 #======================================
 # Add Supervisor configuration file
@@ -149,6 +149,7 @@ ENV SE_BIND_HOST=false \
     SE_REJECT_UNSUPPORTED_CAPS=false \
     SE_OTEL_JAVA_GLOBAL_AUTOCONFIGURE_ENABLED=true \
     SE_OTEL_TRACES_EXPORTER="otlp" \
-    SE_SUPERVISORD_LOG_LEVEL="info"
+    SE_SUPERVISORD_LOG_LEVEL="info" \
+    SE_OPT_BIN="/opt/bin"
 
 CMD ["/opt/bin/entry_point.sh"]

Base/entry_point.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 
+NODE_CONFIG_DIRECTORY=${NODE_CONFIG_DIRECTORY:-$SE_OPT_BIN}
 #==============================================
 # OpenShift or non-sudo environments support
 # https://docs.openshift.com/container-platform/3.11/creating_images/guidelines.html#openshift-specific-guidelines
@@ -17,6 +18,10 @@ SUPERVISOR_PID=$!
 
 function shutdown {
     echo "Trapped SIGTERM/SIGINT/x so shutting down supervisord..."
+    if [ "${SE_NODE_GRACEFUL_SHUTDOWN}" = "true" ]; then
+        echo "Waiting for Selenium Node to shutdown gracefully..."
+        bash ${NODE_CONFIG_DIRECTORY}/nodePreStop.sh
+    fi
     kill -s SIGTERM ${SUPERVISOR_PID}
     wait ${SUPERVISOR_PID}
     echo "Shutdown complete"

Makefile

@@ -68,10 +68,11 @@ build: all
 ci: build test
 
 base:
+	rm -rf ./Base/configs/node && mkdir -p ./Base/configs/node && cp -r ./charts/selenium-grid/configs/node ./Base/configs
 	cd ./Base && docker buildx build --platform $(PLATFORMS) $(BUILD_ARGS) --build-arg VERSION=$(BASE_VERSION) --build-arg RELEASE=$(BASE_RELEASE) --build-arg AUTHORS=$(AUTHORS) -t $(NAME)/base:$(TAG_VERSION) .
 
 base_nightly:
-	cd ./Base && docker buildx build --platform $(PLATFORMS) $(BUILD_ARGS) --build-arg VERSION=$(BASE_VERSION_NIGHTLY) --build-arg RELEASE=$(BASE_RELEASE_NIGHTLY) --build-arg AUTHORS=$(AUTHORS) -t $(NAME)/base:$(TAG_VERSION) .
+	BASE_VERSION=$(BASE_VERSION_NIGHTLY) BASE_RELEASE=$(BASE_RELEASE_NIGHTLY) make base
 
 hub: base
 	cd ./Hub && docker buildx build --platform $(PLATFORMS) $(BUILD_ARGS) $(FROM_IMAGE_ARGS) -t $(NAME)/hub:$(TAG_VERSION) .
@@ -557,7 +558,8 @@ test_parallel: hub chrome firefox edge chromium
 			cd ./tests || true ; \
 			echo TAG=$(TAG_VERSION) > .env ; \
 			echo TEST_DRAIN_AFTER_SESSION_COUNT=$(or $(TEST_DRAIN_AFTER_SESSION_COUNT), 0) >> .env ; \
-			echo TEST_PARALLEL_HARDENING=$(or $(TEST_PARALLEL_HARDENING), "false") >> .env ; \
+			echo TEST_PARALLEL_HARDENING=$(or $(TEST_PARALLEL_HARDENING), "true") >> .env ; \
+			echo TEST_PARALLEL_COUNT=$(or $(TEST_PARALLEL_COUNT), 5) >> .env ; \
 			echo LOG_LEVEL=$(or $(LOG_LEVEL), "INFO") >> .env ; \
 			echo REQUEST_TIMEOUT=$(or $(REQUEST_TIMEOUT), 300) >> .env ; \
 			echo NODE=$$node >> .env ; \
@@ -569,11 +571,14 @@ test_parallel: hub chrome firefox edge chromium
 					echo NODE_CHROME=chromium >> .env ; \
 			fi; \
 			echo TEST_PLATFORMS=$(PLATFORMS) >> .env ; \
-			DOCKER_DEFAULT_PLATFORM=$(PLATFORMS) docker compose --profile $(PLATFORMS) -f docker-compose-v3-test-parallel.yml up --no-log-prefix --exit-code-from tests ; \
-	done
+			export $$(cat .env | xargs) ; \
+			DOCKER_DEFAULT_PLATFORM=$(PLATFORMS) docker compose --profile $(PLATFORMS) -f docker-compose-v3-test-parallel.yml up -d --no-log-prefix ; \
+			RUN_IN_DOCKER_COMPOSE=true bash ./bootstrap.sh $$node ; \
+	done ; \
+	docker compose -f docker-compose-v3-test-parallel.yml down
 
 test_video_dynamic_name:
-	VIDEO_FILE_NAME=auto TEST_DELAY_AFTER_TEST=10 \
+	VIDEO_FILE_NAME=auto TEST_DELAY_AFTER_TEST=0 \
 	make test_video
 
 # This should run on its own CI job. There is no need to combine it with the other tests.
@@ -636,7 +641,7 @@ test_node_relay: hub node_base standalone_firefox
 			echo ANDROID_BASED_IMAGE=$(or $(ANDROID_BASED_IMAGE),docker-android) >> .env ; \
 			echo ANDROID_BASED_TAG=$(or $(ANDROID_BASED_TAG),emulator_14.0) >> .env ; \
 			echo ANDROID_PLATFORM_API=$(or $(ANDROID_PLATFORM_API),14) >> .env ; \
-			echo TEST_DELAY_AFTER_TEST=$(or $(TEST_DELAY_AFTER_TEST), 15) >> .env ; \
+			echo TEST_DELAY_AFTER_TEST=$(or $(TEST_DELAY_AFTER_TEST), 0) >> .env ; \
 			echo NODE=$$node >> .env ; \
 			echo TEST_NODE_RELAY=$$node >> .env ; \
 			echo UID=$$(id -u) >> .env ; \
@@ -688,7 +693,7 @@ test_node_docker: hub standalone_docker standalone_chrome standalone_firefox sta
 			echo LOG_LEVEL=$(or $(LOG_LEVEL), "INFO") >> .env ; \
 			echo REQUEST_TIMEOUT=$(or $(REQUEST_TIMEOUT), 300) >> .env ; \
 			echo SELENIUM_ENABLE_MANAGED_DOWNLOADS=$(or $(SELENIUM_ENABLE_MANAGED_DOWNLOADS), "false") >> .env ; \
-			echo TEST_DELAY_AFTER_TEST=$(or $(TEST_DELAY_AFTER_TEST), 5) >> .env ; \
+			echo TEST_DELAY_AFTER_TEST=$(or $(TEST_DELAY_AFTER_TEST), 0) >> .env ; \
 			echo NODE=$$node >> .env ; \
 			echo UID=$$(id -u) >> .env ; \
 			echo BINDING_VERSION=$(BINDING_VERSION) >> .env ; \
@@ -765,7 +770,7 @@ chart_test_template:
 	./tests/charts/bootstrap.sh
 
 chart_test_autoscaling_disabled:
-	PLATFORMS=$(PLATFORMS) TEST_CHROMIUM=true RELEASE_NAME=selenium SELENIUM_GRID_AUTOSCALING=false TEST_DELAY_AFTER_TEST=15 CHART_ENABLE_TRACING=true \
+	PLATFORMS=$(PLATFORMS) TEST_CHROMIUM=true RELEASE_NAME=selenium SELENIUM_GRID_AUTOSCALING=false TEST_DELAY_AFTER_TEST=0 CHART_ENABLE_TRACING=true \
 	SECURE_INGRESS_ONLY_GENERATE=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_HOST=$$(hostname -i) SELENIUM_GRID_PORT=443 \
 	VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) \
 	./tests/charts/make/chart_test.sh NoAutoscaling

NodeBase/selenium.conf

@@ -57,6 +57,7 @@ stderr_capture_maxbytes=50MB
 priority=15
 command=bash -c "/opt/bin/start-selenium-node.sh; EXIT_CODE=$?; kill -s SIGINT `cat /var/run/supervisor/supervisord.pid`; exit $EXIT_CODE"
 stopasgroup = true
+killasgroup=true
 autostart=true
 autorestart=false
 startsecs=0

NodeChromium/Dockerfile

@@ -7,9 +7,9 @@ LABEL authors=${AUTHORS}
 USER root
 
 # Install Chromium
-RUN apt-get update -qqy \
-  && apt-get install -y software-properties-common \
-  && add-apt-repository ppa:xtradeb/apps \
+RUN echo "deb http://deb.debian.org/debian/ sid main" >> /etc/apt/sources.list \
+  && apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 0E98404D386FA1D9 6ED0E7B82643E131 \
+  && apt-get update -qqy \
   && apt-get -qqy install chromium chromium-l10n \
   && rm -rf /var/lib/apt/lists/* /var/cache/apt/*
 

Video/supervisord.conf

@@ -14,9 +14,11 @@ minprocs=200                                  ; (min. avail process descriptors;
 [program:video-recording]
 priority=0
 command=/opt/bin/video.sh
+stopasgroup = true
+killasgroup=true
 autostart=true
+startsecs=0
 autorestart=true
-stopsignal=INT
 
 ;Logs (all activity redirected to stdout so it can be seen through "docker logs"
 redirect_stderr=true
@@ -28,7 +30,6 @@ priority=5
 command=python3 /opt/bin/video_ready.py
 autostart=true
 autorestart=true
-stopsignal=INT
 
 ;Logs (all activity redirected to stdout so it can be seen through "docker logs"
 redirect_stderr=true
@@ -40,7 +41,6 @@ priority=10
 command=bash -c "if [ ${SE_VIDEO_INTERNAL_UPLOAD} = "true" ]; then /opt/bin/upload.sh; fi"
 autostart=%(ENV_SE_VIDEO_INTERNAL_UPLOAD)s
 autorestart=%(ENV_SE_VIDEO_INTERNAL_UPLOAD)s
-stopsignal=INT
 
 ;Logs (all activity redirected to stdout so it can be seen through "docker logs"
 redirect_stderr=true

charts/selenium-grid/configs/distributor/distributorProbe.sh

@@ -4,7 +4,7 @@ max_time=3
 retry_time=3
 probe_name="Probe.${1:-"Liveness"}"
 ts_format=${SE_LOG_TIMESTAMP_FORMAT:-"+%T.%3N"}
-ROUTER_CONFIG_DIRECTORY=${ROUTER_CONFIG_DIRECTORY:-"/opt/selenium"}
+ROUTER_CONFIG_DIRECTORY=${ROUTER_CONFIG_DIRECTORY:-$SE_OPT_BIN}
 
 GRID_GRAPHQL_URL=$(bash ${ROUTER_CONFIG_DIRECTORY}/routerGraphQLUrl.sh)
 

charts/selenium-grid/configs/node/nodeGridUrl.sh

@@ -2,15 +2,16 @@
 
 max_time=3
 
+if [ -n "${SE_ROUTER_USERNAME}" ] && [ -n "${SE_ROUTER_PASSWORD}" ]; then
+  BASIC_AUTH="${SE_ROUTER_USERNAME}:${SE_ROUTER_PASSWORD}@"
+fi
+if [ "${SE_SUB_PATH}" = "/" ]; then
+  SE_SUB_PATH=""
+fi
+
 if [ -z "${SE_HUB_HOST:-$SE_ROUTER_HOST}" ] || [ -z "${SE_HUB_PORT:-$SE_ROUTER_PORT}" ]; then
   grid_url=""
 else
-  if [ -n "${SE_ROUTER_USERNAME}" ] && [ -n "${SE_ROUTER_PASSWORD}" ]; then
-    BASIC_AUTH="${SE_ROUTER_USERNAME}:${SE_ROUTER_PASSWORD}@"
-  fi
-  if [ "${SE_SUB_PATH}" = "/" ]; then
-    SE_SUB_PATH=""
-  fi
   grid_url=${SE_SERVER_PROTOCOL}://${BASIC_AUTH}${SE_HUB_HOST:-$SE_ROUTER_HOST}:${SE_HUB_PORT:-$SE_ROUTER_PORT}${SE_SUB_PATH}
 fi
 
@@ -19,7 +20,7 @@ if [ -z "${grid_url}" ]; then
 fi
 
 if [ -z "${grid_url}" ]; then
-  return 0
+  grid_url="${SE_SERVER_PROTOCOL}://${BASIC_AUTH}127.0.0.1:4444${SE_SUB_PATH}" # For standalone mode
 fi
 
 grid_url_checks=$(curl --noproxy "*" -m ${max_time} -s -k -o /dev/null -w "%{http_code}" ${grid_url})

charts/selenium-grid/configs/node/nodePreStop.sh

@@ -1,7 +1,8 @@
 #!/bin/bash
 
 probe_name="lifecycle.${1:-"preStop"}"
-NODE_CONFIG_DIRECTORY=${NODE_CONFIG_DIRECTORY:-"/opt/selenium"}
+SE_NODE_PORT=${SE_NODE_PORT:-"5555"}
+NODE_CONFIG_DIRECTORY=${NODE_CONFIG_DIRECTORY:-$SE_OPT_BIN}
 
 max_time=3
 retry_time=5