From 54facfd5edcfa1ad75b7c3019075fcf489002507 Mon Sep 17 00:00:00 2001 From: Emily Stolfo Date: Wed, 12 Jan 2022 14:40:40 +0100 Subject: [PATCH 1/4] Add environment variable for send strategy, background or syncflush --- apm-lambda-extension/extension/process_env.go | 23 +++++++++++++++++++ .../extension/process_env_test.go | 19 +++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/apm-lambda-extension/extension/process_env.go b/apm-lambda-extension/extension/process_env.go index bccc114f..be81be8f 100644 --- a/apm-lambda-extension/extension/process_env.go +++ b/apm-lambda-extension/extension/process_env.go @@ -21,6 +21,7 @@ import ( "log" "os" "strconv" + "strings" ) type extensionConfig struct { @@ -28,9 +29,23 @@ type extensionConfig struct { apmServerSecretToken string apmServerApiKey string dataReceiverServerPort string + sendStrategy SendStrategy dataReceiverTimeoutSeconds int } +// SendStrategy represents the type of sending strategy the extension uses +type SendStrategy string + +const ( + // Background send strategy allows the extension to send remaining buffered + // agent data on the next function invocation + Background SendStrategy = "background" + // SyncFlush send strategy indicates that the extension will synchronously + // flush remaining buffered agent data when it receives a signal that the + // function is complete + SyncFlush SendStrategy = "syncflush" +) + func getIntFromEnv(name string) (int, error) { strValue := os.Getenv(name) value, err := strconv.Atoi(strValue) @@ -54,11 +69,19 @@ func ProcessEnv() *extensionConfig { normalizedApmLambdaServer = normalizedApmLambdaServer + "/" } + // Get the send strategy, convert to lowercase + normalizedSendStrategy := Background + sendStrategy := strings.ToLower(os.Getenv("ELASTIC_APM_SEND_STRATEGY")) + if sendStrategy == string(SyncFlush) { + normalizedSendStrategy = SyncFlush + } + config := &extensionConfig{ apmServerUrl: normalizedApmLambdaServer, apmServerSecretToken: os.Getenv("ELASTIC_APM_SECRET_TOKEN"), apmServerApiKey: os.Getenv("ELASTIC_APM_API_KEY"), dataReceiverServerPort: os.Getenv("ELASTIC_APM_DATA_RECEIVER_SERVER_PORT"), + sendStrategy: normalizedSendStrategy, dataReceiverTimeoutSeconds: dataReceiverTimeoutSeconds, } diff --git a/apm-lambda-extension/extension/process_env_test.go b/apm-lambda-extension/extension/process_env_test.go index 4aec98a4..6c08c423 100644 --- a/apm-lambda-extension/extension/process_env_test.go +++ b/apm-lambda-extension/extension/process_env_test.go @@ -60,6 +60,11 @@ func TestProcessEnv(t *testing.T) { t.Fail() } + if config.sendStrategy != Background { + t.Log("Default send strategy not set correctly") + t.Fail() + } + os.Setenv("ELASTIC_APM_DATA_RECEIVER_SERVER_PORT", ":8201") config = ProcessEnv() if config.dataReceiverServerPort != ":8201" { @@ -87,4 +92,18 @@ func TestProcessEnv(t *testing.T) { t.Log("API Key not set correctly") t.Fail() } + + os.Setenv("ELASTIC_APM_SEND_STRATEGY", "SyncFlush") + config = ProcessEnv() + if config.sendStrategy != "syncflush" { + t.Log("Send strategy not set correctly") + t.Fail() + } + + os.Setenv("ELASTIC_APM_SEND_STRATEGY", "invalid") + config = ProcessEnv() + if config.sendStrategy != "background" { + t.Log("Send strategy not set correctly") + t.Fail() + } } From 2d5bcd37f8eee4aff237c9904d52b484c53ceb15 Mon Sep 17 00:00:00 2001 From: Emily Stolfo Date: Wed, 12 Jan 2022 15:11:11 +0100 Subject: [PATCH 2/4] Only flush at the end of function invocation if send strategy is syncflush --- apm-lambda-extension/extension/process_env.go | 10 +++++----- apm-lambda-extension/extension/process_env_test.go | 13 ++++++++++--- apm-lambda-extension/main.go | 6 ++++-- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/apm-lambda-extension/extension/process_env.go b/apm-lambda-extension/extension/process_env.go index be81be8f..2ee8d575 100644 --- a/apm-lambda-extension/extension/process_env.go +++ b/apm-lambda-extension/extension/process_env.go @@ -29,7 +29,7 @@ type extensionConfig struct { apmServerSecretToken string apmServerApiKey string dataReceiverServerPort string - sendStrategy SendStrategy + SendStrategy SendStrategy dataReceiverTimeoutSeconds int } @@ -70,10 +70,10 @@ func ProcessEnv() *extensionConfig { } // Get the send strategy, convert to lowercase - normalizedSendStrategy := Background + normalizedSendStrategy := SyncFlush sendStrategy := strings.ToLower(os.Getenv("ELASTIC_APM_SEND_STRATEGY")) - if sendStrategy == string(SyncFlush) { - normalizedSendStrategy = SyncFlush + if sendStrategy == string(Background) { + normalizedSendStrategy = Background } config := &extensionConfig{ @@ -81,7 +81,7 @@ func ProcessEnv() *extensionConfig { apmServerSecretToken: os.Getenv("ELASTIC_APM_SECRET_TOKEN"), apmServerApiKey: os.Getenv("ELASTIC_APM_API_KEY"), dataReceiverServerPort: os.Getenv("ELASTIC_APM_DATA_RECEIVER_SERVER_PORT"), - sendStrategy: normalizedSendStrategy, + SendStrategy: normalizedSendStrategy, dataReceiverTimeoutSeconds: dataReceiverTimeoutSeconds, } diff --git a/apm-lambda-extension/extension/process_env_test.go b/apm-lambda-extension/extension/process_env_test.go index 6c08c423..1668add2 100644 --- a/apm-lambda-extension/extension/process_env_test.go +++ b/apm-lambda-extension/extension/process_env_test.go @@ -60,7 +60,7 @@ func TestProcessEnv(t *testing.T) { t.Fail() } - if config.sendStrategy != Background { + if config.SendStrategy != SyncFlush { t.Log("Default send strategy not set correctly") t.Fail() } @@ -95,14 +95,21 @@ func TestProcessEnv(t *testing.T) { os.Setenv("ELASTIC_APM_SEND_STRATEGY", "SyncFlush") config = ProcessEnv() - if config.sendStrategy != "syncflush" { + if config.SendStrategy != "syncflush" { + t.Log("Send strategy not set correctly") + t.Fail() + } + + os.Setenv("ELASTIC_APM_SEND_STRATEGY", "Background") + config = ProcessEnv() + if config.SendStrategy != "background" { t.Log("Send strategy not set correctly") t.Fail() } os.Setenv("ELASTIC_APM_SEND_STRATEGY", "invalid") config = ProcessEnv() - if config.sendStrategy != "background" { + if config.SendStrategy != "syncflush" { t.Log("Send strategy not set correctly") t.Fail() } diff --git a/apm-lambda-extension/main.go b/apm-lambda-extension/main.go index bf77d39f..46deb94a 100644 --- a/apm-lambda-extension/main.go +++ b/apm-lambda-extension/main.go @@ -187,8 +187,10 @@ func main() { log.Println("Time expired waiting for agent signal or runtimeDone event") } - // Flush APM data now that the function invocation has completed - extension.FlushAPMData(client, agentDataChannel, config) + if config.SendStrategy == extension.SyncFlush { + // Flush APM data now that the function invocation has completed + extension.FlushAPMData(client, agentDataChannel, config) + } close(funcDone) close(runtimeDoneSignal) From 91fc05d3091505b36c9e976aaff04611835b0a93 Mon Sep 17 00:00:00 2001 From: Emily Stolfo Date: Wed, 12 Jan 2022 15:16:21 +0100 Subject: [PATCH 3/4] Remove useless test --- apm-lambda-extension/extension/process_env.go | 1 + apm-lambda-extension/extension/process_env_test.go | 7 ------- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/apm-lambda-extension/extension/process_env.go b/apm-lambda-extension/extension/process_env.go index 2ee8d575..be560bec 100644 --- a/apm-lambda-extension/extension/process_env.go +++ b/apm-lambda-extension/extension/process_env.go @@ -40,6 +40,7 @@ const ( // Background send strategy allows the extension to send remaining buffered // agent data on the next function invocation Background SendStrategy = "background" + // SyncFlush send strategy indicates that the extension will synchronously // flush remaining buffered agent data when it receives a signal that the // function is complete diff --git a/apm-lambda-extension/extension/process_env_test.go b/apm-lambda-extension/extension/process_env_test.go index 1668add2..5ad2e9cb 100644 --- a/apm-lambda-extension/extension/process_env_test.go +++ b/apm-lambda-extension/extension/process_env_test.go @@ -93,13 +93,6 @@ func TestProcessEnv(t *testing.T) { t.Fail() } - os.Setenv("ELASTIC_APM_SEND_STRATEGY", "SyncFlush") - config = ProcessEnv() - if config.SendStrategy != "syncflush" { - t.Log("Send strategy not set correctly") - t.Fail() - } - os.Setenv("ELASTIC_APM_SEND_STRATEGY", "Background") config = ProcessEnv() if config.SendStrategy != "background" { From 4954fe617fd05acdcf79c0c9e08c9e1f1b470be6 Mon Sep 17 00:00:00 2001 From: Emily Stolfo Date: Thu, 13 Jan 2022 11:13:27 +0100 Subject: [PATCH 4/4] Add documentation for send strategy env variable --- docs/aws-lambda-extension.asciidoc | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/aws-lambda-extension.asciidoc b/docs/aws-lambda-extension.asciidoc index aab0bbec..40baa911 100644 --- a/docs/aws-lambda-extension.asciidoc +++ b/docs/aws-lambda-extension.asciidoc @@ -149,6 +149,23 @@ If unset, the APM Agent will automatically set the value based on `AWS_LAMBDA_FU The timeout value, in seconds, for the Lambda Extension's server. +[discrete] +[[aws-lambda-send_strategy]] +==== `ELASTIC_APM_SEND_STRATEGY` + +Whether to synchronously flush APM agent data from the extension to the APM server at the end of the function invocation. +The two accepted values are `background` and `syncflush`. The default is `syncflush`. +The `background` strategy indicates that the extension will not flush when it receives a signal that the function invocation +has completed. It will instead send any remaining buffered data on the next function invocation. The result is that, if the +function is not subsequently invoked for that Lambda environment, the buffered data will be lost. However, for lambda functions +that have a steadily frequent load pattern the extension could delay sending the data to the APM server to the next lambda +request and do the sending in parallel to the processing of that next request. This potentially would improve both the lambda +function response time and its throughput. +The other value, `syncflush` will synchronously flush all remaining buffered APM agent data to the APM server when the +extension receives a signal that the function invocation has completed. This strategy blocks the lambda function from receiving +the next request until the extension has flushed all the data. This has a negative effect on the throughput of the function, +though it ensures that all APM data is sent to the APM server. + [discrete] [[aws-lambda-manual-instrumentation]] == Manual Installation