From 1715fa7882de5e28414c19ba4af8443d52c45af6 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Wed, 13 Jul 2022 16:25:51 +0200 Subject: [PATCH 1/4] feat: log a warning if the APM agent data request fails The intake route of the APM server sends a 202 status accepted if the request succeeds. --- apm-lambda-extension/extension/apm_server_transport.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/apm-lambda-extension/extension/apm_server_transport.go b/apm-lambda-extension/extension/apm_server_transport.go index 326eb2b5..316e23f5 100644 --- a/apm-lambda-extension/extension/apm_server_transport.go +++ b/apm-lambda-extension/extension/apm_server_transport.go @@ -185,6 +185,12 @@ func (transport *ApmServerTransport) PostToApmServer(ctx context.Context, agentD return fmt.Errorf("failed to read the response body after posting to the APM server") } + // On success, the server will respond with a 202 Accepted status code. + // Log a warning otherwise. + if resp.StatusCode != http.StatusAccepted { + Log.Warnf("APM server request failed with status code: %d", resp.StatusCode) + } + transport.SetApmServerTransportState(ctx, Healthy) Log.Debug("Transport status set to healthy") Log.Debugf("APM server response body: %v", string(body)) From c93d40f4e66f46e83b6f76041ce1ac9bb65bcac0 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Wed, 13 Jul 2022 16:26:46 +0200 Subject: [PATCH 2/4] test: update test server to reflect APM intake route behaviour --- apm-lambda-extension/extension/apm_server_transport_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apm-lambda-extension/extension/apm_server_transport_test.go b/apm-lambda-extension/extension/apm_server_transport_test.go index eab30a57..500091a9 100644 --- a/apm-lambda-extension/extension/apm_server_transport_test.go +++ b/apm-lambda-extension/extension/apm_server_transport_test.go @@ -60,10 +60,12 @@ func TestPostToApmServerDataCompressed(t *testing.T) { bytes, _ := ioutil.ReadAll(r.Body) assert.Equal(t, string(data), string(bytes)) assert.Equal(t, "gzip", r.Header.Get("Content-Encoding")) + w.WriteHeader(http.StatusAccepted) if _, err := w.Write([]byte(`{"foo": "bar"}`)); err != nil { t.Fail() return } + })) defer apmServer.Close() @@ -105,6 +107,7 @@ func TestPostToApmServerDataNotCompressed(t *testing.T) { compressedBytes, _ := ioutil.ReadAll(pr) assert.Equal(t, string(compressedBytes), string(requestBytes)) assert.Equal(t, "gzip", r.Header.Get("Content-Encoding")) + w.WriteHeader(http.StatusAccepted) if _, err := w.Write([]byte(`{"foo": "bar"}`)); err != nil { t.Fail() return @@ -334,6 +337,7 @@ func TestAPMServerRecovery(t *testing.T) { bytes, _ := ioutil.ReadAll(r.Body) assert.Equal(t, string(data), string(bytes)) assert.Equal(t, "gzip", r.Header.Get("Content-Encoding")) + w.WriteHeader(http.StatusAccepted) if _, err := w.Write([]byte(`{"foo": "bar"}`)); err != nil { return } From bb356d3ea13e67e91b3db6af74638a7f9f263ed0 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Wed, 13 Jul 2022 16:50:30 +0200 Subject: [PATCH 3/4] fix: update code to only handle auth errors (401) --- apm-lambda-extension/extension/apm_server_transport.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apm-lambda-extension/extension/apm_server_transport.go b/apm-lambda-extension/extension/apm_server_transport.go index 316e23f5..9e33e2cc 100644 --- a/apm-lambda-extension/extension/apm_server_transport.go +++ b/apm-lambda-extension/extension/apm_server_transport.go @@ -185,10 +185,10 @@ func (transport *ApmServerTransport) PostToApmServer(ctx context.Context, agentD return fmt.Errorf("failed to read the response body after posting to the APM server") } - // On success, the server will respond with a 202 Accepted status code. - // Log a warning otherwise. - if resp.StatusCode != http.StatusAccepted { - Log.Warnf("APM server request failed with status code: %d", resp.StatusCode) + if resp.StatusCode == http.StatusUnauthorized { + Log.Warnf("Authentication with the APM server failed: response status code: %d", resp.StatusCode) + Log.Debugf("APM server response body: %v", string(body)) + return nil } transport.SetApmServerTransportState(ctx, Healthy) From 0155a3b0f8981e4f70a244f5189cda1c66152137 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Wed, 13 Jul 2022 16:51:26 +0200 Subject: [PATCH 4/4] fix: add test to assert the transport status after auth error --- .../extension/apm_server_transport_test.go | 47 ++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/apm-lambda-extension/extension/apm_server_transport_test.go b/apm-lambda-extension/extension/apm_server_transport_test.go index 500091a9..09a1f8ce 100644 --- a/apm-lambda-extension/extension/apm_server_transport_test.go +++ b/apm-lambda-extension/extension/apm_server_transport_test.go @@ -65,7 +65,6 @@ func TestPostToApmServerDataCompressed(t *testing.T) { t.Fail() return } - })) defer apmServer.Close() @@ -363,6 +362,52 @@ func TestAPMServerRecovery(t *testing.T) { assert.Equal(t, transport.reconnectionCount, -1) } +func TestAPMServerAuthFails(t *testing.T) { + // Compress the data + pr, pw := io.Pipe() + gw, _ := gzip.NewWriterLevel(pw, gzip.BestSpeed) + go func() { + if _, err := gw.Write([]byte("")); err != nil { + t.Fail() + return + } + if err := gw.Close(); err != nil { + t.Fail() + return + } + if err := pw.Close(); err != nil { + t.Fail() + return + } + }() + + // Create AgentData struct with compressed data + data, _ := io.ReadAll(pr) + agentData := AgentData{Data: data, ContentEncoding: "gzip"} + + // Create apm server and handler + apmServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + })) + defer apmServer.Close() + + config := extensionConfig{ + apmServerUrl: apmServer.URL + "/", + } + + transport := InitApmServerTransport(&config) + transport.SetApmServerTransportState(context.Background(), Healthy) + transport.SetApmServerTransportState(context.Background(), Failing) + for { + if transport.status != Failing { + break + } + } + assert.Equal(t, transport.status, Pending) + assert.NoError(t, transport.PostToApmServer(context.Background(), agentData)) + assert.NotEqual(t, transport.status, Healthy) +} + func TestContinuedAPMServerFailure(t *testing.T) { // Compress the data pr, pw := io.Pipe()