Experimental: Selenium Grid scaler with different nodeMaxSessions per node (#2402)

VietND96 · web-flow · commit 0ac93a9cf3f4 · 2024-09-21T07:19:59.000+07:00
Signed-off-by: Viet Nguyen Duc &lt;nguyenducviet4496@gmail.com&gt;
diff --git a/.keda/README.md b/.keda/README.md
@@ -20,7 +20,7 @@ docker pull selenium/keda-admission-webhooks:2.15.1-selenium-grid-20240907
 
 Besides that, you also can use image tag `latest` or `nightly`.
 
-If you are deploying KEDA core using their official Helm chart, you can overwrite the image registry and tag by providing the following values in the `values.yaml` file. For example:
+If you are deploying KEDA core using their official Helm [chart](https://github.com/kedacore/charts), you can overwrite the image registry and tag by providing the following values in the `values.yaml` file. For example:
 
 ```yaml
   image:
diff --git a/Makefile b/Makefile
@@ -913,7 +913,7 @@ chart_test_autoscaling_disabled:
 chart_test_autoscaling_deployment_https:
 	PLATFORMS=$(PLATFORMS) CHART_FULL_DISTRIBUTED_MODE=true CHART_ENABLE_BASIC_AUTH=true \
 	SECURE_INGRESS_ONLY_DEFAULT=true INGRESS_DISABLE_USE_HTTP2=true SELENIUM_GRID_PROTOCOL=https CHART_ENABLE_INGRESS_HOSTNAME=true SELENIUM_GRID_PORT=443 \
-	SELENIUM_GRID_AUTOSCALING_MIN_REPLICA=1 \
+	SELENIUM_GRID_AUTOSCALING_MIN_REPLICA=1 MAX_SESSIONS_FIREFOX=3 MAX_SESSIONS_EDGE=2 MAX_SESSIONS_CHROME=1 \
 	VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) KEDA_BASED_NAME=$(KEDA_BASED_NAME) KEDA_BASED_TAG=$(KEDA_BASED_TAG) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) \
 	TEMPLATE_OUTPUT_FILENAME="k8s_fullDistributed_basicAuth_secureIngress_defaultCerts_ingressHostName_disableHttp2_autoScaling_scaledObject_subPath.yaml" \
 	./tests/charts/make/chart_test.sh DeploymentAutoscaling
@@ -929,6 +929,7 @@ chart_test_autoscaling_deployment:
 chart_test_autoscaling_job_https:
 	PLATFORMS=$(PLATFORMS) TEST_EXISTING_KEDA=true RELEASE_NAME=selenium CHART_ENABLE_BASIC_AUTH=true \
 	SECURE_CONNECTION_SERVER=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_PORT=443 SUB_PATH=/ \
+	MAX_SESSIONS_FIREFOX=1 MAX_SESSIONS_EDGE=2 MAX_SESSIONS_CHROME=3 \
 	VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) KEDA_BASED_NAME=$(KEDA_BASED_NAME) KEDA_BASED_TAG=$(KEDA_BASED_TAG) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) EXTERNAL_UPLOADER_CONFIG=true \
 	TEMPLATE_OUTPUT_FILENAME="k8s_prefixSelenium_basicAuth_secureServer_autoScaling_scaledJob_existingKEDA.yaml" \
 	./tests/charts/make/chart_test.sh JobAutoscaling
diff --git a/Video/video.sh b/Video/video.sh
@@ -210,7 +210,7 @@ else
       if [[ "${endpoint_status}" = "401" ]]; then
         echo "$(date +%FT%T%Z) [${process_name}] - GraphQL endpoint requires authentication, please set env variables SE_ROUTER_USERNAME and SE_ROUTER_PASSWORD"
       elif [[ "${endpoint_status}" = "404" ]]; then
-        echo "$(date +%FT%T%Z) [${process_name}] -GraphQL endpoint could not be found, please check the endpoint ${endpoint_url}"
+        echo "$(date +%FT%T%Z) [${process_name}] - GraphQL endpoint could not be found, please check the endpoint ${endpoint_url}"
       fi
       echo "$(date +%FT%T%Z) [${process_name}] - Start recording: $caps_se_video_record, video file name: $video_file_name"
       log_node_response
diff --git a/charts/selenium-grid/CONFIGURATION.md b/charts/selenium-grid/CONFIGURATION.md
@@ -32,7 +32,7 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
 | global.seleniumGrid.imageTag | string | `"4.24.0-20240907"` | Image tag for all selenium components |
 | global.seleniumGrid.nodesImageTag | string | `"4.24.0-20240907"` | Image tag for browser's nodes |
 | global.seleniumGrid.videoImageTag | string | `"ffmpeg-7.0.2-20240907"` | Image tag for browser's video recorder |
-| global.seleniumGrid.kubectlImage | string | `"jitesoft/kubectl:latest"` | kubectl image is used to execute kubectl commands in utility jobs |
+| global.seleniumGrid.kubectlImage | string | `"bitnami/kubectl:latest"` | kubectl image is used to execute kubectl commands in utility jobs |
 | global.seleniumGrid.imagePullSecret | string | `""` | Pull secret for all components, can be overridden individually |
 | global.seleniumGrid.logLevel | string | `"INFO"` | Log level for all components. Possible values describe here: https://www.selenium.dev/documentation/grid/configuration/cli_options/#logging |
 | global.seleniumGrid.defaultNodeStartupProbe | string | `"exec"` | Set default startup probe method for all nodes (supplied values: httpGet, exec). If not set, the default is httpGet |
@@ -46,6 +46,7 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
 | global.seleniumGrid.updateStrategy.rollingUpdate | object | `{"maxSurge":1,"maxUnavailable":0}` | Specify for strategy RollingUpdate |
 | global.seleniumGrid.affinity | object | `{}` | Specify affinity for all components, can be overridden individually |
 | global.seleniumGrid.topologySpreadConstraints | list | `[]` | Specify topologySpreadConstraints for all components, can be overridden individually |
+| global.seleniumGrid.nodeMaxSessions | int | `1` | Specify number of max sessions per node. Can be overridden by individual component (this is also set to scaler trigger parameter `nodeMaxSessions` if `autoscaling` is enabled) |
 | tls.nameOverride | string | `nil` | Name of external secret containing the TLS certificate and key |
 | tls.enabled | bool | `false` | Enable or disable TLS for the server components (and ingress proxy) |
 | tls.ingress.enabled | bool | `false` | Enable or disable TLS for the ingress proxy only |
@@ -362,6 +363,7 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
 | chromeNode.lifecycle | object | `{}` | Define postStart and preStop events. This overwrites the defined preStop in deregisterLifecycle if any |
 | chromeNode.extraVolumeMounts | list | `[]` | Extra volume mounts for chrome-node container |
 | chromeNode.extraVolumes | list | `[]` | Extra volumes for chrome-node pod |
+| chromeNode.nodeMaxSessions | string | `nil` | Override the number of max sessions per node |
 | chromeNode.scaledOptions | string | `nil` | Override the scaled options for chrome nodes |
 | chromeNode.scaledJobOptions | string | `nil` | Override the scaledJobOptions for chrome nodes |
 | chromeNode.scaledObjectOptions | string | `nil` | Override the scaledObjectOptions for chrome nodes |
@@ -411,6 +413,7 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
 | firefoxNode.lifecycle | object | `{}` | Define postStart and preStop events. This overwrites the defined preStop in deregisterLifecycle if any |
 | firefoxNode.extraVolumeMounts | list | `[]` | Extra volume mounts for firefox-node container |
 | firefoxNode.extraVolumes | list | `[]` | Extra volumes for firefox-node pod |
+| firefoxNode.nodeMaxSessions | string | `nil` | Override the number of max sessions per node |
 | firefoxNode.scaledOptions | string | `nil` | Override the scaled options for firefox nodes |
 | firefoxNode.scaledJobOptions | string | `nil` | Override the scaledJobOptions for firefox nodes |
 | firefoxNode.scaledObjectOptions | string | `nil` | Override the scaledObjectOptions for firefox nodes |
@@ -460,6 +463,7 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
 | edgeNode.lifecycle | object | `{}` | Define postStart and preStop events. This overwrites the defined preStop in deregisterLifecycle if any |
 | edgeNode.extraVolumeMounts | list | `[]` | Extra volume mounts for edge-node container |
 | edgeNode.extraVolumes | list | `[]` | Extra volumes for edge-node pod |
+| edgeNode.nodeMaxSessions | string | `nil` | Override the number of max sessions per node |
 | edgeNode.scaledOptions | string | `nil` | Override the scaled options for edge nodes |
 | edgeNode.scaledJobOptions | string | `nil` | Override the scaledJobOptions for edge nodes |
 | edgeNode.scaledObjectOptions | string | `nil` | Override the scaledObjectOptions for edge nodes |
diff --git a/charts/selenium-grid/templates/_helpers.tpl b/charts/selenium-grid/templates/_helpers.tpl
@@ -207,6 +207,7 @@ Common autoscaling spec template
 */}}
 {{- define "seleniumGrid.autoscalingTemplate" -}}
 {{- $spec := toYaml (dict) -}}
+{{- $nodeMaxSessions := default $.Values.global.seleniumGrid.nodeMaxSessions .node.nodeMaxSessions | int64 -}}
 {{/* Merge with precedence from right to left */}}
 {{- with $.Values.autoscaling.scaledOptions -}}
   {{- $spec = mergeOverwrite ($spec | fromYaml) . | toYaml -}}
@@ -239,7 +240,7 @@ Common autoscaling spec template
 triggers:
   - type: selenium-grid
     metadata:
-      triggerIndex: '{{ default $.Values.autoscaling.scaledOptions.minReplicaCount (.node.scaledOptions).minReplicaCount }}'
+      nodeMaxSessions: {{ $nodeMaxSessions | quote }}
   {{- with .node.hpa }}
     {{- tpl (toYaml .) $ | nindent 6 }}
   {{- end }}
@@ -278,6 +279,7 @@ Common pod template
 {{- $nodeImageTag := default $.Values.global.seleniumGrid.nodesImageTag .node.imageTag -}}
 {{- $videoImageRegistry := default $.Values.global.seleniumGrid.imageRegistry $.Values.videoRecorder.imageRegistry -}}
 {{- $videoImageTag := default $.Values.global.seleniumGrid.videoImageTag $.Values.videoRecorder.imageTag -}}
+{{- $nodeMaxSessions := default $.Values.global.seleniumGrid.nodeMaxSessions .node.nodeMaxSessions | int64 -}}
 template:
   metadata:
     labels:
@@ -329,6 +331,14 @@ template:
         image: {{ printf "%s/%s:%s" $nodeImageRegistry .node.imageName $nodeImageTag }}
         imagePullPolicy: {{ .node.imagePullPolicy }}
         env:
+          - name: SE_NODE_MAX_SESSIONS
+            value: {{ $nodeMaxSessions | quote }}
+        {{- if gt $nodeMaxSessions 1 }}
+          - name: SE_NODE_OVERRIDE_MAX_SESSIONS
+            value: "true"
+        {{- end }}
+          - name: SE_DRAIN_AFTER_SESSION_COUNT
+            value: {{ and (eq (include "seleniumGrid.useKEDA" $) "true") (eq .Values.autoscaling.scalingType "job") | ternary $nodeMaxSessions 0 | quote }}
           - name: SE_NODE_CONTAINER_NAME
             valueFrom:
               fieldRef:
@@ -469,6 +479,10 @@ template:
         image: {{ printf "%s/%s:%s" $videoImageRegistry $.Values.videoRecorder.imageName $videoImageTag }}
         imagePullPolicy: {{ $.Values.videoRecorder.imagePullPolicy }}
         env:
+        - name: SE_NODE_MAX_SESSIONS
+          value: {{ $nodeMaxSessions | quote }}
+        - name: SE_DRAIN_AFTER_SESSION_COUNT
+          value: {{ and (eq (include "seleniumGrid.useKEDA" $) "true") (eq .Values.autoscaling.scalingType "job") | ternary $nodeMaxSessions 0 | quote }}
         - name: SE_NODE_PORT
           value: {{ .node.port | quote }}
         - name: DISPLAY_CONTAINER_NAME
diff --git a/charts/selenium-grid/templates/node-configmap.yaml b/charts/selenium-grid/templates/node-configmap.yaml
@@ -23,7 +23,6 @@ data:
 {{- end }}
   NODE_CONFIG_DIRECTORY: '{{ $.Values.nodeConfigMap.extraScriptsDirectory }}'
   SE_SUB_PATH: '{{ template "seleniumGrid.url.subPath" $ }}'
-  SE_DRAIN_AFTER_SESSION_COUNT: '{{- and (eq (include "seleniumGrid.useKEDA" .) "true") (eq .Values.autoscaling.scalingType "job") | ternary "1" "0" -}}'
 {{- if $.Values.videoRecorder.enabled }}
   SE_VIDEO_CONTAINER_NAME: {{ $.Values.videoRecorder.name | quote }}
 {{- end }}
diff --git a/charts/selenium-grid/values.yaml b/charts/selenium-grid/values.yaml
@@ -13,7 +13,7 @@ global:
     # -- Image tag for browser's video recorder
     videoImageTag: ffmpeg-7.0.2-20240907
     # -- kubectl image is used to execute kubectl commands in utility jobs
-    kubectlImage: jitesoft/kubectl:latest
+    kubectlImage: bitnami/kubectl:latest
     # -- Pull secret for all components, can be overridden individually
     imagePullSecret: ""
     # -- Log level for all components. Possible values describe here: https://www.selenium.dev/documentation/grid/configuration/cli_options/#logging
@@ -48,6 +48,8 @@ global:
     #    topologyKey: kubernetes.io/hostname
     #    whenUnsatisfiable: DoNotSchedule
     #    Note: If not define labelSelector, it will be added automatically based on "app" label in each component
+    # -- Specify number of max sessions per node. Can be overridden by individual component (this is also set to scaler trigger parameter `nodeMaxSessions` if `autoscaling` is enabled)
+    nodeMaxSessions: 1
 
 tls:
   # -- Name of external secret containing the TLS certificate and key
@@ -1006,6 +1008,8 @@ chromeNode:
   #   persistentVolumeClaim:
   #     claimName: my-pv-claim
 
+  # -- Override the number of max sessions per node
+  nodeMaxSessions:
   # -- Override the scaled options for chrome nodes
   scaledOptions:
   # -- Override the scaledJobOptions for chrome nodes
@@ -1185,6 +1189,8 @@ firefoxNode:
   #   persistentVolumeClaim:
   #     claimName: my-pv-claim
 
+  # -- Override the number of max sessions per node
+  nodeMaxSessions:
   # -- Override the scaled options for firefox nodes
   scaledOptions:
   # -- Override the scaledJobOptions for firefox nodes
@@ -1363,6 +1369,8 @@ edgeNode:
   #   persistentVolumeClaim:
   #     claimName: my-pv-claim
 
+  # -- Override the number of max sessions per node
+  nodeMaxSessions:
   # -- Override the scaled options for edge nodes
   scaledOptions:
   # -- Override the scaledJobOptions for edge nodes
diff --git a/tests/charts/make/chart_test.sh b/tests/charts/make/chart_test.sh
@@ -49,6 +49,9 @@ else
 fi
 EXTERNAL_TLS_SECRET_NAME=${EXTERNAL_TLS_SECRET_NAME:-"external-tls-secret"}
 SELENIUM_ENABLE_MANAGED_DOWNLOADS=${SELENIUM_ENABLE_MANAGED_DOWNLOADS:-"true"}
+MAX_SESSIONS_CHROME=${MAX_SESSIONS_CHROME:-"1"}
+MAX_SESSIONS_FIREFOX=${MAX_SESSIONS_FIREFOX:-"1"}
+MAX_SESSIONS_EDGE=${MAX_SESSIONS_EDGE:-"1"}
 
 cleanup() {
   # Get the list of pods
@@ -126,6 +129,9 @@ HELM_COMMAND_SET_IMAGES=" \
 --set global.seleniumGrid.httpLogs=${CHART_ENABLE_TRACING} \
 --set isolateComponents=${CHART_FULL_DISTRIBUTED_MODE} \
 --set global.seleniumGrid.logLevel=${LOG_LEVEL} \
+--set chromeNode.nodeMaxSessions=${MAX_SESSIONS_CHROME} \
+--set firefoxNode.nodeMaxSessions=${MAX_SESSIONS_FIREFOX} \
+--set edgeNode.nodeMaxSessions=${MAX_SESSIONS_EDGE} \
 "
 
 if [ "${SELENIUM_GRID_AUTOSCALING}" = "true" ] && [ "${TEST_EXISTING_KEDA}" = "true" ]; then
diff --git a/tests/charts/templates/render/dummy.yaml b/tests/charts/templates/render/dummy.yaml
@@ -17,6 +17,7 @@ global:
       - maxSkew: 4
         topologyKey: kubernetes.io/hostname
         whenUnsatisfiable: DoNotSchedule
+    nodeMaxSessions: 3
 
 autoscaling:
   enableWithExistingKEDA: true
@@ -95,11 +96,13 @@ components:
     serviceType: NodePort
 
 chromeNode:
+  nodeMaxSessions: 2
   annotations:
     "restartOnUpdate": "true"
   terminationGracePeriodSeconds: 7200
 
 firefoxNode:
+  nodeMaxSessions: 1
   annotations:
     "restartOnUpdate": "true"
   terminationGracePeriodSeconds: 720
diff --git a/tests/charts/templates/render/dummy_solution.yaml b/tests/charts/templates/render/dummy_solution.yaml
@@ -17,6 +17,7 @@ global:
       - maxSkew: 4
         topologyKey: kubernetes.io/hostname
         whenUnsatisfiable: DoNotSchedule
+    nodeMaxSessions: 3
 
 selenium-grid:
   autoscaling:
@@ -86,10 +87,12 @@ selenium-grid:
       serviceType: NodePort
 
   chromeNode:
+    nodeMaxSessions: 2
     affinity: *affinity
     terminationGracePeriodSeconds: 7200
 
   firefoxNode:
+    nodeMaxSessions: 1
     affinity: *affinity
     terminationGracePeriodSeconds: 720
 
diff --git a/tests/charts/templates/test.py b/tests/charts/templates/test.py
@@ -329,6 +329,34 @@ def test_router_envFrom_secretRef_name_use_external_secret_when_basicAuth_nameOv
                             is_present = True
         self.assertTrue(is_present, "ENV variable from secretRef name is not set to external secret")
 
+    def test_scaler_triggers_authenticationRef_name_is_added(self):
+        resources_name = ['{0}selenium-chrome-node'.format(RELEASE_NAME),
+                          '{0}selenium-edge-node'.format(RELEASE_NAME),
+                          '{0}selenium-firefox-node'.format(RELEASE_NAME),]
+        is_present = False
+        for doc in LIST_OF_DOCUMENTS:
+            if doc['metadata']['name'] in resources_name and doc['kind'] == 'ScaledObject':
+                logger.info(f"Assert authenticationRef name is added to scaler triggers")
+                name = doc['spec']['triggers'][0]['authenticationRef']['name']
+                self.assertTrue(name, '{0}selenium-scaler-trigger-auth'.format(RELEASE_NAME))
+
+    def test_scaler_triggers_parameter_nodeMaxSessions_global_and_individual_value(self):
+        resources_name = {'{0}selenium-chrome-node'.format(RELEASE_NAME): 2,
+                          '{0}selenium-edge-node'.format(RELEASE_NAME): 3,
+                          '{0}selenium-firefox-node'.format(RELEASE_NAME): 1,}
+        count = 0
+        for resource_name in resources_name.keys():
+            for doc in LIST_OF_DOCUMENTS:
+                if doc['metadata']['name'] == resource_name and doc['kind'] == 'ScaledObject':
+                    logger.info(f"Assert nodeMaxSessions parameter is set in scaler triggers")
+                    self.assertTrue(doc['spec']['triggers'][0]['metadata']['nodeMaxSessions'] == str(resources_name[doc['metadata']['name']]))
+                if doc['metadata']['name'] == resource_name and doc['kind'] == 'Deployment':
+                    for env in doc['spec']['template']['spec']['containers'][0]['env']:
+                        if env['name'] == 'SE_NODE_MAX_SESSIONS':
+                            self.assertTrue(env['value'] == str(resources_name[doc['metadata']['name']]), "Value is not matched")
+                            count += 1
+        self.assertEqual(count, len(resources_name.keys()), "Expected {0} resources but found {1}".format(len(resources_name.keys()), count))
+
 if __name__ == '__main__':
     failed = False
     try:
