End-to-end testing using SAM CLI and locally run Lambdas

.gitignore

@@ -11,5 +11,11 @@
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
 
+# Goland config files
+*.idea
+
+# MacOS DS_Store files
+*.DS_Store
+
 # Dependency directories (remove the comment below to include it)
 # vendor/

apm-lambda-extension/e2e-testing/.e2e_test_config

@@ -0,0 +1,2 @@
+RUN_E2E_TESTS=false
+DEBUG_OUTPUT=false

apm-lambda-extension/e2e-testing/.gitignore

@@ -0,0 +1,5 @@
+**/.aws-sam
+**/sam-java/agent
+**/sam-java/*/.gradle
+**/sam-java/*/gradle
+**/.env

apm-lambda-extension/e2e-testing/README.md

@@ -0,0 +1,34 @@
+# End-to-End Testing
+
+The file `e2e_test.go` contains an end-to-end test of the Elastic APM Lambda Extension. This test is built on top of the AWS SAM CLI, which allows running Lambda functions and their associated layers locally.
+
+## Setup
+
+Since this test is sensibly longer than the other unit tests, it is disabled by default. To enable it, go to `.e2e_test_config` and set the environment variable `RUN_E2E_TESTS` to `true`.
+In order to run the Lambda functions locally, the following dependencies must be installed : 
+- [Install](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/serverless-sam-cli-install.html) the SAM CLI. Creating an AWS account is actually not required.
+- Install Docker
+- Install a Go Runtime
+
+## Run
+
+```shell
+cd apm-lambda-extension/e2e-testing
+go test
+```
+
+### Command line arguments 
+The command line arguments are presented with their default value.
+```shell
+-rebuild=false          # Rebuilds the Lambda function images
+-lang=nodejs            # Selects the language of the Lambda function. node, java and python are supported.
+-timer=20               # The timeout (in seconds) used to stop the execution of the Lambda function.
+                        # Recommended values : NodeJS : 20, Python : 30, Java : 40
+-java-agent-ver=1.28.4  # The version of the Java agent used when Java is selected.
+```
+
+Example :
+```shell
+go test -rebuild=false -lang=java -timer=40 -java-agent-ver=1.28.4
+```
+

apm-lambda-extension/e2e-testing/e2e_test.go

@@ -0,0 +1,291 @@
+package e2e_testing
+
+import (
+	"archive/zip"
+	"bufio"
+	"bytes"
+	"compress/gzip"
+	"compress/zlib"
+	"errors"
+	"flag"
+	"fmt"
+	"github.com/google/uuid"
+	"github.com/joho/godotenv"
+	"github.com/stretchr/testify/assert"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+var rebuildPtr = flag.Bool("rebuild", false, "rebuild lambda functions")
+var langPtr = flag.String("lang", "nodejs", "the language of the Lambda test function : Java, Node, or Python")
+var timerPtr = flag.Int("timer", 20, "the timeout of the test lambda function")
+var javaAgentVerPtr = flag.String("java-agent-ver", "1.28.4", "the version of the java APM agent")
+
+func TestEndToEnd(t *testing.T) {
+
+	// Check the only mandatory environment variable
+	if err := godotenv.Load(".e2e_test_config"); err != nil {
+		log.Println("No additional .e2e_test_config file found")
+	}
+	if getEnvVarValueOrSetDefault("RUN_E2E_TESTS", "false") != "true" {
+		t.Skip("Skipping E2E tests. Please set the env. variable RUN_E2E_TESTS=true if you want to run them.")
+	}
+
+	languageName := strings.ToLower(*langPtr)
+	supportedLanguages := []string{"nodejs", "python", "java"}
+	if !isStringInSlice(languageName, supportedLanguages) {
+		processError(errors.New(fmt.Sprintf("Unsupported language %s ! Supported languages are %v", languageName, supportedLanguages)))
+	}
+
+	samPath := "sam-" + languageName
+	samServiceName := "sam-testing-" + languageName
+
+	// Build and download required binaries (extension and Java agent)
+	buildExtensionBinaries()
+
+	// Java agent processing
+	if languageName == "java" {
+		if !folderExists(filepath.Join(samPath, "agent")) {
+			log.Println("Java agent not found ! Collecting archive from Github...")
+			retrieveJavaAgent(samPath, *javaAgentVerPtr)
+		}
+		changeJavaAgentPermissions(samPath)
+	}
+
+	// Initialize Mock APM Server
+	mockAPMServerLog := ""
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.RequestURI == "/intake/v2/events" {
+			bytesRes, _ := getDecompressedBytesFromRequest(r)
+			mockAPMServerLog += fmt.Sprintf("%s\n", bytesRes)
+		}
+	}))
+	defer ts.Close()
+
+	resultsChan := make(chan string, 1)
+
+	uuid := runTestWithTimer(samPath, samServiceName, ts.URL, *rebuildPtr, *timerPtr, resultsChan)
+	log.Printf("UUID generated during the test : %s", uuid)
+	if uuid == "" {
+		t.Fail()
+	}
+	log.Printf("Querying the mock server for transaction bound to %s...", samServiceName)
+	assert.True(t, strings.Contains(mockAPMServerLog, uuid))
+}
+
+func runTestWithTimer(path string, serviceName string, serverURL string, buildFlag bool, lambdaFuncTimeout int, resultsChan chan string) string {
+	timer := time.NewTimer(time.Duration(lambdaFuncTimeout) * time.Second * 2)
+	defer timer.Stop()
+	go runTest(path, serviceName, serverURL, buildFlag, lambdaFuncTimeout, resultsChan)
+	select {
+	case uuid := <-resultsChan:
+		return uuid
+	case <-timer.C:
+		return ""
+	}
+}
+
+func buildExtensionBinaries() {
+	runCommandInDir("make", []string{}, "..", getEnvVarValueOrSetDefault("DEBUG_OUTPUT", "false") == "true")
+}
+
+func runTest(path string, serviceName string, serverURL string, buildFlag bool, lambdaFuncTimeout int, resultsChan chan string) {
+	log.Printf("Starting to test %s", serviceName)
+
+	if !folderExists(filepath.Join(path, ".aws-sam")) || buildFlag {
+		log.Printf("Building the Lambda function %s", serviceName)
+		runCommandInDir("sam", []string{"build"}, path, getEnvVarValueOrSetDefault("DEBUG_OUTPUT", "false") == "true")
+	}
+
+	log.Printf("Invoking the Lambda function %s", serviceName)
+	uuidWithHyphen := uuid.New().String()
+	urlSlice := strings.Split(serverURL, ":")
+	port := urlSlice[len(urlSlice)-1]
+	runCommandInDir("sam", []string{"local", "invoke", "--parameter-overrides",
+		fmt.Sprintf("ParameterKey=ApmServerURL,ParameterValue=http://host.docker.internal:%s", port),
+		fmt.Sprintf("ParameterKey=TestUUID,ParameterValue=%s", uuidWithHyphen),
+		fmt.Sprintf("ParameterKey=TimeoutParam,ParameterValue=%d", lambdaFuncTimeout)},
+		path, getEnvVarValueOrSetDefault("DEBUG_OUTPUT", "false") == "true")
+	log.Printf("%s execution complete", serviceName)
+
+	resultsChan <- uuidWithHyphen
+}
+
+func retrieveJavaAgent(samJavaPath string, version string) {
+
+	agentFolderPath := filepath.Join(samJavaPath, "agent")
+	agentArchivePath := filepath.Join(samJavaPath, "agent.zip")
+
+	// Download archive
+	out, err := os.Create(agentArchivePath)
+	processError(err)
+	defer out.Close()
+	resp, err := http.Get(fmt.Sprintf("https://github.com/elastic/apm-agent-java/releases/download/v%[1]s/elastic-apm-java-aws-lambda-layer-%[1]s.zip", version))
+	processError(err)
+	defer resp.Body.Close()
+	io.Copy(out, resp.Body)
+
+	// Unzip archive and delete it
+	log.Println("Unzipping Java Agent archive...")
+	unzip(agentArchivePath, agentFolderPath)
+	err = os.Remove(agentArchivePath)
+	processError(err)
+}
+
+func changeJavaAgentPermissions(samJavaPath string) {
+	agentFolderPath := filepath.Join(samJavaPath, "agent")
+	log.Println("Setting appropriate permissions for Java agent files...")
+	agentFiles, err := ioutil.ReadDir(agentFolderPath)
+	processError(err)
+	for _, f := range agentFiles {
+		os.Chmod(filepath.Join(agentFolderPath, f.Name()), 0755)
+	}
+}
+
+func getEnvVarValueOrSetDefault(envVarName string, defaultVal string) string {
+	val := os.Getenv(envVarName)
+	if val == "" {
+		return defaultVal
+	}
+	return val
+}
+
+func runCommandInDir(command string, args []string, dir string, printOutput bool) {
+	e := exec.Command(command, args...)
+	if printOutput {
+		log.Println(e.String())
+	}
+	e.Dir = dir
+	stdout, _ := e.StdoutPipe()
+	stderr, _ := e.StderrPipe()
+	e.Start()
+	scannerOut := bufio.NewScanner(stdout)
+	for scannerOut.Scan() {
+		m := scannerOut.Text()
+		if printOutput {
+			log.Println(m)
+		}
+	}
+	scannerErr := bufio.NewScanner(stderr)
+	for scannerErr.Scan() {
+		m := scannerErr.Text()
+		if printOutput {
+			log.Println(m)
+		}
+	}
+	e.Wait()
+
+}
+
+func folderExists(path string) bool {
+	_, err := os.Stat(path)
+	if err == nil {
+		return true
+	}
+	return false
+}
+
+func processError(err error) {
+	if err != nil {
+		log.Panic(err)
+	}
+}
+
+func unzip(archivePath string, destinationFolderPath string) {
+
+	openedArchive, err := zip.OpenReader(archivePath)
+	processError(err)
+	defer openedArchive.Close()
+
+	// Permissions setup
+	os.MkdirAll(destinationFolderPath, 0755)
+
+	// Closure required, so that Close() calls do not pile up when unzipping archives with a lot of files
+	extractAndWriteFile := func(f *zip.File) error {
+		rc, err := f.Open()
+		if err != nil {
+			return err
+		}
+		defer func() {
+			if err := rc.Close(); err != nil {
+				panic(err)
+			}
+		}()
+
+		path := filepath.Join(destinationFolderPath, f.Name)
+
+		// Check for ZipSlip (Directory traversal)
+		if !strings.HasPrefix(path, filepath.Clean(destinationFolderPath)+string(os.PathSeparator)) {
+			return fmt.Errorf("illegal file path: %s", path)
+		}
+
+		if f.FileInfo().IsDir() {
+			os.MkdirAll(path, f.Mode())
+		} else {
+			os.MkdirAll(filepath.Dir(path), f.Mode())
+			f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
+			processError(err)
+			defer f.Close()
+			_, err = io.Copy(f, rc)
+			processError(err)
+		}
+		return nil
+	}
+
+	for _, f := range openedArchive.File {
+		err := extractAndWriteFile(f)
+		processError(err)
+	}
+}
+
+func getDecompressedBytesFromRequest(req *http.Request) ([]byte, error) {
+	var rawBytes []byte
+	if req.Body != nil {
+		rawBytes, _ = ioutil.ReadAll(req.Body)
+	}
+
+	switch req.Header.Get("Content-Encoding") {
+	case "deflate":
+		reader := bytes.NewReader([]byte(rawBytes))
+		zlibreader, err := zlib.NewReader(reader)
+		if err != nil {
+			return nil, fmt.Errorf("could not create zlib.NewReader: %v", err)
+		}
+		bodyBytes, err := ioutil.ReadAll(zlibreader)
+		if err != nil {
+			return nil, fmt.Errorf("could not read from zlib reader using ioutil.ReadAll: %v", err)
+		}
+		return bodyBytes, nil
+	case "gzip":
+		reader := bytes.NewReader([]byte(rawBytes))
+		zlibreader, err := gzip.NewReader(reader)
+		if err != nil {
+			return nil, fmt.Errorf("could not create gzip.NewReader: %v", err)
+		}
+		bodyBytes, err := ioutil.ReadAll(zlibreader)
+		if err != nil {
+			return nil, fmt.Errorf("could not read from gzip reader using ioutil.ReadAll: %v", err)
+		}
+		return bodyBytes, nil
+	default:
+		return rawBytes, nil
+	}
+}
+
+func isStringInSlice(a string, list []string) bool {
+	for _, b := range list {
+		if b == a {
+			return true
+		}
+	}
+	return false
+}

apm-lambda-extension/e2e-testing/sam-java/sam-testing-java/build.gradle

@@ -0,0 +1,14 @@
+plugins {
+    id 'java'
+}
+
+repositories {
+    mavenCentral()
+}
+
+dependencies {
+    implementation 'com.amazonaws:aws-lambda-java-core:1.2.1'
+    implementation 'com.amazonaws:aws-lambda-java-events:3.6.0'
+    testImplementation 'junit:junit:4.13.1'
+    implementation "co.elastic.apm:apm-agent-api:1.28.4"
+}