Test Chart installation for Plus and remote images (#2060)

lucacome · web-flow · commit 118488e9b624 · 2024-05-31T15:15:19.000-07:00
Problem: We're not testing the Helm Chart installation with NGINX Plus
and with pre-built images

Solution: Add installation steps for both
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -31,20 +31,33 @@ jobs:
     outputs:
       go_path: ${{ steps.vars.outputs.go_path }}
       min_k8s_version: ${{ steps.vars.outputs.min_k8s_version }}
+      k8s_latest: ${{ steps.vars.outputs.k8s_latest }}
+      helm_changes: ${{ steps.filter.outputs.charts }}
     steps:
       - name: Checkout Repository
         uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+        with:
+          fetch-depth: 0
 
       - name: Setup Golang Environment
         uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
         with:
           go-version: stable
 
+      - name: Check for changes
+        uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
+        id: filter
+        with:
+          filters: |
+            charts:
+            - charts/nginx-gateway-fabric/**/*
+
       - name: Output Variables
         id: vars
         run: |
           echo "go_path=$(go env GOPATH)" >> $GITHUB_OUTPUT
-          echo "min_k8s_version=1.25.16" >> $GITHUB_OUTPUT
+          echo "min_k8s_version=v1.25.16" >> $GITHUB_OUTPUT
+          echo "k8s_latest=$(grep -m1 'FROM kindest/node' <tests/Dockerfile | awk -F'[:]' '{print $2}')" >> $GITHUB_OUTPUT
 
       - name: Check if go.mod and go.sum are up to date
         run: go mod tidy && git diff --exit-code -- go.mod go.sum
@@ -211,7 +224,7 @@ jobs:
       fail-fast: false
       matrix:
         image: [nginx, plus]
-        k8s-version: ["${{ needs.vars.outputs.min_k8s_version }}", "latest"]
+        k8s-version: ["${{ needs.vars.outputs.min_k8s_version }}", "${{ needs.vars.outputs.k8s_latest }}"]
     uses: ./.github/workflows/functional.yml
     with:
       image: ${{ matrix.image }}
@@ -226,7 +239,7 @@ jobs:
       fail-fast: false
       matrix:
         image: [nginx, plus]
-        k8s-version: ["${{ needs.vars.outputs.min_k8s_version }}", "latest"]
+        k8s-version: ["${{ needs.vars.outputs.min_k8s_version }}", "${{ needs.vars.outputs.k8s_latest }}"]
         enable-experimental: [true, false]
     uses: ./.github/workflows/conformance.yml
     with:
@@ -238,96 +251,18 @@ jobs:
 
   helm-tests:
     name: Helm Tests
-    runs-on: ubuntu-22.04
-    needs: [vars, build-oss]
-    steps:
-      - name: Checkout Repository
-        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
-
-      - name: Fetch Cached Artifacts
-        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
-        with:
-          path: ${{ github.workspace }}/dist
-          key: nginx-gateway-fabric-${{ github.run_id }}-${{ github.run_number }}
-
-      - name: Docker Buildx
-        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0
-
-      - name: NGF Docker meta
-        id: ngf-meta
-        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
-        with:
-          images: |
-            name=ghcr.io/nginxinc/nginx-gateway-fabric
-          tags: |
-            type=semver,pattern={{version}}
-            type=edge
-            type=ref,event=pr
-            type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }}
-
-      - name: NGINX Docker meta
-        id: nginx-meta
-        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
-        with:
-          images: |
-            name=ghcr.io/nginxinc/nginx-gateway-fabric/nginx
-          tags: |
-            type=semver,pattern={{version}}
-            type=edge
-            type=ref,event=pr
-            type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }}
-
-      - name: Build NGF Docker Image
-        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
-        with:
-          file: build/Dockerfile
-          tags: ${{ steps.ngf-meta.outputs.tags }}
-          context: "."
-          target: goreleaser
-          load: true
-          cache-from: type=gha,scope=ngf
-          pull: true
-
-      - name: Build NGINX Docker Image
-        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
-        with:
-          file: build/Dockerfile.nginx
-          tags: ${{ steps.nginx-meta.outputs.tags }}
-          context: "."
-          load: true
-          cache-from: type=gha,scope=nginx
-          pull: true
-          build-args: |
-            NJS_DIR=internal/mode/static/nginx/modules/src
-            NGINX_CONF_DIR=internal/mode/static/nginx/conf
-            BUILD_AGENT=gha
-
-      - name: Deploy Kubernetes
-        id: k8s
-        run: |
-          kube_config=${{ github.workspace }}/kube-${{ github.run_id }}-helm
-          make create-kind-cluster KIND_KUBE_CONFIG=${kube_config}
-          echo "KUBECONFIG=${kube_config}" >> "$GITHUB_ENV"
-          kind load docker-image ghcr.io/nginxinc/nginx-gateway-fabric:${{ steps.ngf-meta.outputs.version }} ghcr.io/nginxinc/nginx-gateway-fabric/nginx:${{ steps.nginx-meta.outputs.version }}
-          kubectl kustomize config/crd/gateway-api/standard | kubectl apply -f -
-
-      - name: Install Chart
-        run: >
-          helm install
-          helm-$(echo ${{ steps.ngf-meta.outputs.version }} | tr '.' '-')
-          .
-          --wait
-          --create-namespace
-          --set nginxGateway.image.repository=ghcr.io/nginxinc/nginx-gateway-fabric
-          --set nginxGateway.image.tag=${{ steps.ngf-meta.outputs.version }}
-          --set nginxGateway.image.pullPolicy=Never
-          --set nginxGateway.productTelemetry.enable=false
-          --set nginx.image.repository=ghcr.io/nginxinc/nginx-gateway-fabric/nginx
-          --set nginx.image.tag=${{ steps.nginx-meta.outputs.version }}
-          --set nginx.image.pullPolicy=Never
-          --set service.type=NodePort
-          -n nginx-gateway
-        working-directory: ${{ github.workspace }}/charts/nginx-gateway-fabric
+    needs: [vars, build-oss, build-plus]
+    strategy:
+      fail-fast: false
+      matrix:
+        image: [nginx, plus]
+        k8s-version: ["${{ needs.vars.outputs.min_k8s_version }}", "${{ needs.vars.outputs.k8s_latest }}"]
+    uses: ./.github/workflows/helm.yml
+    with:
+      image: ${{ matrix.image }}
+      k8s-version: ${{ matrix.k8s-version }}
+    secrets: inherit
+    if: ${{ needs.vars.outputs.helm_changes == 'true' || github.event_name == 'schedule' }}
 
   publish-helm:
     name: Package and Publish Helm Chart
diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml
@@ -129,23 +129,22 @@ jobs:
       - name: Deploy Kubernetes
         id: k8s
         run: |
-          k8s_version=${{ inputs.k8s-version }}
-          make create-kind-cluster KIND_KUBE_CONFIG=${{ github.workspace }}/kube-${{ github.run_id }} ${{ ! contains(inputs.k8s-version, 'latest') && 'KIND_IMAGE=kindest/node:v${k8s_version}' || '' }}
-          echo "KUBECONFIG=${{ github.workspace }}/kube-${{ github.run_id }}" >> "$GITHUB_ENV"
-        working-directory: ./tests
+          kind create cluster --name ${{ github.run_id }} --image=kindest/node:${{ inputs.k8s-version }}
+          kind load docker-image ${{ join(fromJSON(steps.ngf-meta.outputs.json).tags, ' ') }} ${{ join(fromJSON(steps.nginx-meta.outputs.json).tags, ' ') }} --name ${{ github.run_id }}
 
       - name: Setup conformance tests
         run: |
           ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric
           ngf_tag=${{ steps.ngf-meta.outputs.version }}
           if [ ${{ github.event_name }} == "schedule" ]; then export GW_API_VERSION=main; fi
           if [ ${{ inputs.enable-experimental }} == "true" ]; then export ENABLE_EXPERIMENTAL=true; fi
-          make install-ngf-local-no-build${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag}
+          make helm-install-local${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag}
+          make deploy-updated-provisioner PREFIX=${ngf_prefix} TAG=${ngf_tag}
         working-directory: ./tests
 
       - name: Run conformance tests
         run: |
-          make run-conformance-tests CONFORMANCE_TAG=${{ github.sha }} NGF_VERSION=${{ github.ref_name }}
+          make run-conformance-tests CONFORMANCE_TAG=${{ github.sha }} NGF_VERSION=${{ github.ref_name }} CLUSTER_NAME=${{ github.run_id }}
           core_result=$(cat conformance-profile.yaml | yq '.profiles[0].core.result')
           extended_result=$(cat conformance-profile.yaml | yq '.profiles[0].extended.result')
           if [ "${core_result}" == "failure" ] || [ "${extended_result}" == "failure" ]; then echo "Conformance test failed, see above for details." && exit 2; fi
diff --git a/.github/workflows/functional.yml b/.github/workflows/functional.yml
@@ -101,17 +101,8 @@ jobs:
       - name: Deploy Kubernetes
         id: k8s
         run: |
-          k8s_version=${{ inputs.k8s-version }}
-          make create-kind-cluster KIND_KUBE_CONFIG=${{ github.workspace }}/kube-${{ github.run_id }} ${{ ! contains(inputs.k8s-version, 'latest') && 'KIND_IMAGE=kindest/node:v${k8s_version}' || '' }}
-          echo "KUBECONFIG=${{ github.workspace }}/kube-${{ github.run_id }}" >> "$GITHUB_ENV"
-
-      - name: Setup functional tests
-        id: setup
-        run: |
-          ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric
-          ngf_tag=${{ steps.ngf-meta.outputs.version }}
-          make load-images${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag}
-        working-directory: ./tests
+          kind create cluster --name ${{ github.run_id }} --image=kindest/node:${{ inputs.k8s-version }}
+          kind load docker-image ${{ join(fromJSON(steps.ngf-meta.outputs.json).tags, ' ') }} ${{ join(fromJSON(steps.nginx-meta.outputs.json).tags, ' ') }} --name ${{ github.run_id }}
 
       - name: Run functional telemetry tests
         run: |
diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml
@@ -0,0 +1,166 @@
+name: Helm Testing
+
+on:
+  workflow_call:
+    inputs:
+      image:
+        required: true
+        type: string
+      k8s-version:
+        required: true
+        type: string
+
+permissions:
+  contents: read
+
+jobs:
+  helm-tests-local:
+    name: Helm Tests Local
+    runs-on: ubuntu-22.04
+    if: ${{ github.event_name != 'schedule' }}
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+        with:
+          fetch-depth: 0
+
+      - name: Fetch Cached Artifacts
+        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+        with:
+          path: ${{ github.workspace }}/dist
+          key: nginx-gateway-fabric-${{ github.run_id }}-${{ github.run_number }}
+
+      - name: Docker Buildx
+        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0
+
+      - name: NGF Docker meta
+        id: ngf-meta
+        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
+        with:
+          images: |
+            name=ghcr.io/nginxinc/nginx-gateway-fabric
+          tags: |
+            type=semver,pattern={{version}}
+            type=edge
+            type=schedule
+            type=ref,event=pr
+            type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }}
+
+      - name: NGINX Docker meta
+        id: nginx-meta
+        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
+        with:
+          images: |
+            name=ghcr.io/nginxinc/nginx-gateway-fabric/${{ inputs.image == 'plus' && 'nginx-plus' || inputs.image }}
+          tags: |
+            type=semver,pattern={{version}}
+            type=edge
+            type=schedule
+            type=ref,event=pr
+            type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }}
+
+      - name: Build NGF Docker Image
+        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
+        with:
+          file: build/Dockerfile
+          tags: ${{ steps.ngf-meta.outputs.tags }}
+          context: "."
+          target: goreleaser
+          load: true
+          cache-from: type=gha,scope=ngf
+          pull: true
+
+      - name: Build NGINX Docker Image
+        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
+        with:
+          file: build/Dockerfile${{ inputs.image == 'nginx' && '.nginx' || '' }}${{ inputs.image == 'plus' && '.nginxplus' || ''}}
+          tags: ${{ steps.nginx-meta.outputs.tags }}
+          context: "."
+          load: true
+          cache-from: type=gha,scope=${{ inputs.image }}
+          pull: true
+          build-args: |
+            NJS_DIR=internal/mode/static/nginx/modules/src
+            NGINX_CONF_DIR=internal/mode/static/nginx/conf
+            BUILD_AGENT=gha
+
+      - name: Install cloud-provider-kind
+        run: |
+          curl -fsSL -O https://github.com/kubernetes-sigs/cloud-provider-kind/releases/download/v0.2.0/cloud-provider-kind_0.2.0_linux_amd64.tar.gz
+          tar -xvf cloud-provider-kind_0.2.0_linux_amd64.tar.gz
+
+      - name: Run cloud-provider-kind
+        run: ./cloud-provider-kind & > cloud-provider-kind.log 2>&1
+
+      - name: Deploy Kubernetes
+        id: k8s
+        run: |
+          kind create cluster --name ${{ github.run_id }} --image=kindest/node:${{ inputs.k8s-version }}
+          kind load docker-image ${{ join(fromJSON(steps.ngf-meta.outputs.json).tags, ' ') }} ${{ join(fromJSON(steps.nginx-meta.outputs.json).tags, ' ') }} --name ${{ github.run_id }}
+          kubectl kustomize config/crd/gateway-api/standard | kubectl apply -f -
+
+      - name: Set up Python
+        uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
+        with:
+          python-version: "3.x"
+          check-latest: true
+
+      - name: Set up chart-testing
+        uses: helm/chart-testing-action@e6669bcd63d7cb57cb4380c33043eebe5d111992 # v2.6.1
+
+      - name: Install Chart
+        run: |
+          ct install --config .ct.yaml --helm-extra-set-args="--set=nginxGateway.image.tag=${{ steps.ngf-meta.outputs.version }} \
+          --set=nginx.image.repository=ghcr.io/nginxinc/nginx-gateway-fabric/nginx${{ inputs.image == 'plus' && '-plus' || ''}} \
+          --set=nginx.plus=${{ inputs.image == 'plus' }} \
+          --set=nginx.image.tag=${{ steps.nginx-meta.outputs.version }} \
+          --set=nginxGateway.image.pullPolicy=Never \
+          --set=nginx.image.pullPolicy=Never \
+          --set=nginxGateway.productTelemetry.enable=false"
+
+  helm-test-remote:
+    name: Helm Tests Remote
+    runs-on: ubuntu-22.04
+    if: ${{ github.event_name == 'schedule' }}
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+        with:
+          fetch-depth: 0
+
+      - name: Install cloud-provider-kind
+        run: |
+          curl -fsSL -O https://github.com/kubernetes-sigs/cloud-provider-kind/releases/download/v0.2.0/cloud-provider-kind_0.2.0_linux_amd64.tar.gz
+          tar -xvf cloud-provider-kind_0.2.0_linux_amd64.tar.gz
+
+      - name: Run cloud-provider-kind
+        run: ./cloud-provider-kind & > cloud-provider-kind.log 2>&1
+
+      - name: Deploy Kubernetes
+        id: k8s
+        run: |
+          kind create cluster --name ${{ github.run_id }} --image=kindest/node:${{ inputs.k8s-version }}
+          kubectl kustomize config/crd/gateway-api/standard | kubectl apply -f -
+          kubectl create namespace nginx-gateway
+
+      - name: Create k8s secret
+        if: ${{ inputs.image == 'plus' }}
+        run: |
+          kubectl create secret docker-registry nginx-plus-registry-secret --docker-server=private-registry.nginx.com --docker-username=${{ secrets.JWT_PLUS_REGISTRY }} --docker-password=none -n nginx-gateway
+
+      - name: Set up Python
+        uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
+        with:
+          python-version: "3.x"
+          check-latest: true
+
+      - name: Set up chart-testing
+        uses: helm/chart-testing-action@e6669bcd63d7cb57cb4380c33043eebe5d111992 # v2.6.1
+
+      - name: Install Chart
+        run: |
+          ct install --config .ct.yaml --all --namespace nginx-gateway --helm-extra-set-args="--set=nginxGateway.image.tag=nightly \
+          --set=nginx.plus=${{ inputs.image == 'plus' }} \
+          --set=nginx.image.tag=nightly \
+          --set=nginxGateway.productTelemetry.enable=false \
+          ${{ inputs.image == 'plus' && '--set=serviceAccount.imagePullSecret=nginx-plus-registry-secret --set=nginx.image.repository=private-registry.nginx.com/nginx-gateway-fabric/nginx-plus' || '' }}"
diff --git a/Makefile b/Makefile
@@ -31,7 +31,6 @@ NGINX_PREFIX ?= $(PREFIX)/nginx## The name of the nginx image. For example: ngin
 NGINX_PLUS_PREFIX ?= $(PREFIX)/nginx-plus## The name of the nginx plus image. For example: nginx-gateway-fabric/nginx-plus
 TAG ?= $(VERSION:v%=%)## The tag of the image. For example, 1.1.0
 TARGET ?= local## The target of the build. Possible values: local and container
-KIND_KUBE_CONFIG=$${HOME}/.kube/kind/config## The location of the kind kubeconfig
 OUT_DIR ?= build/out## The folder where the binary will be stored
 GOARCH ?= amd64## The architecture of the image and/or binary. For example: amd64 or arm64
 GOOS ?= linux## The OS of the image and/or binary. For example: linux or darwin
@@ -149,7 +148,6 @@ deps: ## Add missing and remove unused modules, verify deps and download them to
 create-kind-cluster: ## Create a kind cluster
 	$(eval KIND_IMAGE=$(shell grep -m1 'FROM kindest/node' <$(SELF_DIR)tests/Dockerfile | awk -F'[ ]' '{print $$2}'))
 	kind create cluster --image $(KIND_IMAGE)
-	kind export kubeconfig --kubeconfig $(KIND_KUBE_CONFIG)
 
 .PHONY: delete-kind-cluster
 delete-kind-cluster: ## Delete kind cluster
diff --git a/tests/Makefile b/tests/Makefile
