Add basic system test with utilities (#1274)

Problem: In order to test a full NGF system running in k8s, we need a framework that can easily deploy apps and send traffic.

Solution: Enhance the framework with functions to create apps and send traffic using port forwarding. Also added a basic test to utilize these functions as a proof of concept.

Author: sjberman

Makefile

@@ -123,7 +123,7 @@ lint: ## Run golangci-lint against code
 
 .PHONY: unit-test
 unit-test: ## Run unit tests for the go code
-	go test ./... -tags unit -race -coverprofile cover.out
+	go test ./internal/... -race -coverprofile cover.out
 	go tool cover -html=cover.out -o cover.html
 
 .PHONY: njs-unit-test

tests/Makefile

@@ -22,7 +22,7 @@ load-images: ## Load NGF and NGINX images on configured kind cluster
 	kind load docker-image $(PREFIX):$(TAG) $(NGINX_PREFIX):$(TAG)
 
 test: ## Run the system tests against your default k8s cluster
-	go test -v . -args --gateway-api-version=$(GW_API_VERSION) --image-tag=$(TAG) \
+	go test -v ./suite -args --gateway-api-version=$(GW_API_VERSION) --image-tag=$(TAG) \
 		--ngf-image-repo=$(PREFIX) --nginx-image-repo=$(NGINX_PREFIX) --pull-policy=$(PULL_POLICY) \
 		--k8s-version=$(K8S_VERSION)
 

tests/README.md

@@ -0,0 +1,119 @@
+# System Testing
+
+The tests in this directory are meant to be run on a live Kubernetes environment to verify a real system. These
+are similar to the existing [conformance tests](../conformance/README.md), but will verify things such as:
+
+- NGF-specific functionality
+- Non-Functional requirements testing (such as performance, scale, etc.)
+
+When running, the tests create a port-forward from your NGF Pod to localhost using a port chosen by the
+test framework. Traffic is sent over this port.
+
+Directory structure is as follows:
+
+- `framework`: contains utility functions for running the tests
+- `suite`: contains the test files
+
+**Note**: Existing NFR tests will be migrated into this testing `suite` and results stored in a `results` directory.
+
+## Prerequisites
+
+- Kubernetes cluster.
+- Docker.
+- Golang.
+
+**Note**: all commands in steps below are executed from the `tests` directory
+
+```shell
+make
+```
+
+```text
+build-images                   Build NGF and NGINX images
+create-kind-cluster            Create a kind cluster
+delete-kind-cluster            Delete kind cluster
+help                           Display this help
+load-images                    Load NGF and NGINX images on configured kind cluster
+test                           Run the system tests against your default k8s cluster
+```
+
+**Note:** The following variables are configurable when running the below `make` commands:
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| TAG      | edge    | tag for the locally built NGF images |
+| PREFIX   | nginx-gateway-fabric | prefix for the locally built NGF image |
+| NGINX_PREFIX | nginx-gateway-fabric/nginx | prefix for the locally built NGINX image |
+| PULL_POLICY | Never | NGF image pull policy |
+| GW_API_VERSION | 1.0.0 | version of Gateway API resources to install |
+| K8S_VERSION | latest | version of k8s that the tests are run on |
+
+## Step 1 - Create a Kubernetes cluster
+
+This can be done in a cloud provider of choice, or locally using `kind`:
+
+```makefile
+make create-kind-cluster
+```
+
+> Note: The default kind cluster deployed is the latest available version. You can specify a different version by
+> defining the kind image to use through the KIND_IMAGE variable, e.g.
+
+```makefile
+make create-kind-cluster KIND_IMAGE=kindest/node:v1.27.3
+```
+
+## Step 2 - Build and Load Images
+
+Loading the images only applies to a `kind` cluster. If using a cloud provider, you will need to tag and push
+your images to a registry that is accessible from that cloud provider.
+
+```makefile
+make build-images load-images TAG=$(whoami)
+```
+
+## Step 3 - Run the tests
+
+```makefile
+make test TAG=$(whoami)
+```
+
+To run a specific test, you can "focus" it by adding the `F` prefix to the name. For example:
+
+```go
+It("runs some test", func(){
+    ...
+})
+```
+
+becomes:
+
+```go
+FIt("runs some test", func(){
+    ...
+})
+```
+
+This can also be done at higher levels like `Context`.
+
+To disable a specific test, add the `X` prefix to it, similar to the previous example:
+
+```go
+It("runs some test", func(){
+    ...
+})
+```
+
+becomes:
+
+```go
+XIt("runs some test", func(){
+    ...
+})
+```
+
+## Step 4 - Delete kind cluster
+
+```makefile
+make delete-kind-cluster
+```

tests/framework/portforward.go

@@ -0,0 +1,91 @@
+package framework
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/url"
+	"path"
+	"time"
+
+	core "k8s.io/api/core/v1"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/portforward"
+	"k8s.io/client-go/transport/spdy"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// GetNGFPodName returns the name of the NGF Pod.
+func GetNGFPodName(
+	k8sClient client.Client,
+	namespace,
+	releaseName string,
+	timeout time.Duration,
+) (string, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	var podList core.PodList
+	if err := k8sClient.List(
+		ctx,
+		&podList,
+		client.InNamespace(namespace),
+		client.MatchingLabels{
+			"app.kubernetes.io/instance": releaseName,
+		},
+	); err != nil {
+		return "", fmt.Errorf("error getting list of Pods: %w", err)
+	}
+
+	if len(podList.Items) > 0 {
+		return podList.Items[0].Name, nil
+	}
+
+	return "", errors.New("unable to find NGF Pod")
+}
+
+// PortForward starts a port-forward to the specified Pod and returns the local port being forwarded.
+func PortForward(config *rest.Config, namespace, podName string, stopCh chan struct{}) (int, error) {
+	roundTripper, upgrader, err := spdy.RoundTripperFor(config)
+	if err != nil {
+		return 0, fmt.Errorf("error creating roundtripper: %w", err)
+	}
+
+	serverURL, err := url.Parse(config.Host)
+	if err != nil {
+		return 0, fmt.Errorf("error parsing rest config host: %w", err)
+	}
+
+	serverURL.Path = path.Join(
+		"api", "v1",
+		"namespaces", namespace,
+		"pods", podName,
+		"portforward",
+	)
+
+	dialer := spdy.NewDialer(upgrader, &http.Client{Transport: roundTripper}, http.MethodPost, serverURL)
+
+	readyCh := make(chan struct{}, 1)
+	out, errOut := new(bytes.Buffer), new(bytes.Buffer)
+
+	forwarder, err := portforward.New(dialer, []string{":80"}, stopCh, readyCh, out, errOut)
+	if err != nil {
+		return 0, fmt.Errorf("error creating port forwarder: %w", err)
+	}
+
+	go func() {
+		if err := forwarder.ForwardPorts(); err != nil {
+			panic(err)
+		}
+	}()
+
+	<-readyCh
+	ports, err := forwarder.GetPorts()
+	if err != nil {
+		return 0, fmt.Errorf("error getting ports being forwarded: %w", err)
+	}
+
+	return int(ports[0].Local), nil
+}

tests/framework/request.go

@@ -0,0 +1,50 @@
+package framework
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+	"strings"
+	"time"
+)
+
+// Get sends a GET request to the specified url.
+// It resolves to localhost (where the NGF port-forward is running) instead of using DNS.
+// The status and body of the response is returned, or an error.
+func Get(url string, timeout time.Duration) (int, string, error) {
+	dialer := &net.Dialer{}
+
+	http.DefaultTransport.(*http.Transport).DialContext = func(
+		ctx context.Context,
+		network,
+		addr string,
+	) (net.Conn, error) {
+		split := strings.Split(addr, ":")
+		port := split[len(split)-1]
+		return dialer.DialContext(ctx, network, fmt.Sprintf("127.0.0.1:%s", port))
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return 0, "", err
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return 0, "", err
+	}
+	defer resp.Body.Close()
+
+	body := new(bytes.Buffer)
+	_, err = body.ReadFrom(resp.Body)
+	if err != nil {
+		return resp.StatusCode, "", err
+	}
+
+	return resp.StatusCode, body.String(), nil
+}