diff --git a/apm-lambda-extension/NOTICE.txt b/apm-lambda-extension/NOTICE.txt index 9bf9a42c..79ec13bf 100644 --- a/apm-lambda-extension/NOTICE.txt +++ b/apm-lambda-extension/NOTICE.txt @@ -285,6 +285,217 @@ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +-------------------------------------------------------------------------------- +Module : go.elastic.co/apm/v2 +Version : v2.1.0 +Time : 2022-05-20T09:45:34Z +Licence : Apache-2.0 + +Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/v2@v2.1.0/LICENSE: + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2018 Elasticsearch BV + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + -------------------------------------------------------------------------------- Module : go.elastic.co/ecszap Version : v1.0.1 @@ -496,6 +707,39 @@ Contents of probable licence file $GOMODCACHE/go.elastic.co/ecszap@v1.0.1/LICENS See the License for the specific language governing permissions and limitations under the License. +-------------------------------------------------------------------------------- +Module : go.elastic.co/fastjson +Version : v1.1.0 +Time : 2020-05-11T07:15:19Z +Licence : MIT + +Contents of probable licence file $GOMODCACHE/go.elastic.co/fastjson@v1.1.0/LICENSE: + +Copyright 2018 Elasticsearch BV + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +--- + +Copyright (c) 2016 Mail.Ru Group + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + -------------------------------------------------------------------------------- Module : go.uber.org/atomic Version : v1.9.0 diff --git a/apm-lambda-extension/dependencies.asciidoc b/apm-lambda-extension/dependencies.asciidoc index 6db74eb3..a7629211 100644 --- a/apm-lambda-extension/dependencies.asciidoc +++ b/apm-lambda-extension/dependencies.asciidoc @@ -17,7 +17,9 @@ This page lists the third-party dependencies used to build {n}. | link:https://github.com/aws/aws-sdk-go[$$github.com/aws/aws-sdk-go$$] | v1.44.27 | Apache-2.0 | link:https://github.com/jmespath/go-jmespath[$$github.com/jmespath/go-jmespath$$] | v0.4.0 | Apache-2.0 | link:https://github.com/pkg/errors[$$github.com/pkg/errors$$] | v0.9.1 | BSD-2-Clause +| link:https://go.elastic.co/apm/v2[$$go.elastic.co/apm/v2$$] | v2.1.0 | Apache-2.0 | link:https://go.elastic.co/ecszap[$$go.elastic.co/ecszap$$] | v1.0.1 | Apache-2.0 +| link:https://go.elastic.co/fastjson[$$go.elastic.co/fastjson$$] | v1.1.0 | MIT | link:https://go.uber.org/atomic[$$go.uber.org/atomic$$] | v1.9.0 | MIT | link:https://go.uber.org/multierr[$$go.uber.org/multierr$$] | v1.8.0 | MIT | link:https://go.uber.org/zap[$$go.uber.org/zap$$] | v1.21.0 | MIT diff --git a/apm-lambda-extension/e2e-testing/e2e_util.go b/apm-lambda-extension/e2e-testing/e2e_util.go index 485c097a..c88f3089 100644 --- a/apm-lambda-extension/e2e-testing/e2e_util.go +++ b/apm-lambda-extension/e2e-testing/e2e_util.go @@ -20,10 +20,6 @@ package e2eTesting import ( "archive/zip" "bufio" - "bytes" - "compress/gzip" - "compress/zlib" - "elastic/apm-lambda-extension/extension" "fmt" "io" "io/ioutil" @@ -33,6 +29,8 @@ import ( "os/exec" "path/filepath" "strings" + + "elastic/apm-lambda-extension/extension" ) // GetEnvVarValueOrSetDefault retrieves the environment variable envVarName. @@ -157,33 +155,7 @@ func GetDecompressedBytesFromRequest(req *http.Request) ([]byte, error) { if req.Body != nil { rawBytes, _ = ioutil.ReadAll(req.Body) } - - switch req.Header.Get("Content-Encoding") { - case "deflate": - reader := bytes.NewReader(rawBytes) - zlibreader, err := zlib.NewReader(reader) - if err != nil { - return nil, fmt.Errorf("could not create zlib.NewReader: %v", err) - } - bodyBytes, err := ioutil.ReadAll(zlibreader) - if err != nil { - return nil, fmt.Errorf("could not read from zlib reader using ioutil.ReadAll: %v", err) - } - return bodyBytes, nil - case "gzip": - reader := bytes.NewReader(rawBytes) - zlibreader, err := gzip.NewReader(reader) - if err != nil { - return nil, fmt.Errorf("could not create gzip.NewReader: %v", err) - } - bodyBytes, err := ioutil.ReadAll(zlibreader) - if err != nil { - return nil, fmt.Errorf("could not read from gzip reader using ioutil.ReadAll: %v", err) - } - return bodyBytes, nil - default: - return rawBytes, nil - } + return extension.GetUncompressedBytes(rawBytes, req.Header.Get("Content-Encoding")) } // GetFreePort is a function that queries the kernel and obtains an unused port. diff --git a/apm-lambda-extension/extension/apm_server_transport.go b/apm-lambda-extension/extension/apm_server_transport.go index 9b519098..326eb2b5 100644 --- a/apm-lambda-extension/extension/apm_server_transport.go +++ b/apm-lambda-extension/extension/apm_server_transport.go @@ -75,7 +75,7 @@ func InitApmServerTransport(config *extensionConfig) *ApmServerTransport { // StartBackgroundApmDataForwarding Receive agent data as it comes in and post it to the APM server. // Stop checking for, and sending agent data when the function invocation // has completed, signaled via a channel. -func (transport *ApmServerTransport) ForwardApmData(ctx context.Context) error { +func (transport *ApmServerTransport) ForwardApmData(ctx context.Context, metadataContainer *MetadataContainer) error { if transport.status == Failing { return nil } @@ -85,6 +85,13 @@ func (transport *ApmServerTransport) ForwardApmData(ctx context.Context) error { Log.Debug("Invocation context cancelled, not processing any more agent data") return nil case agentData := <-transport.dataChannel: + if metadataContainer.Metadata == nil { + metadata, err := ProcessMetadata(agentData) + if err != nil { + Log.Errorf("Error extracting metadata from agent payload %v", err) + } + metadataContainer.Metadata = metadata + } if err := transport.PostToApmServer(ctx, agentData); err != nil { return fmt.Errorf("error sending to APM server, skipping: %v", err) } diff --git a/apm-lambda-extension/extension/apm_server_transport_test.go b/apm-lambda-extension/extension/apm_server_transport_test.go index 5362e588..eab30a57 100644 --- a/apm-lambda-extension/extension/apm_server_transport_test.go +++ b/apm-lambda-extension/extension/apm_server_transport_test.go @@ -20,12 +20,13 @@ package extension import ( "compress/gzip" "context" - "github.com/stretchr/testify/assert" "io" "io/ioutil" "net/http" "net/http/httptest" "testing" + + "github.com/stretchr/testify/assert" ) func TestPostToApmServerDataCompressed(t *testing.T) { diff --git a/apm-lambda-extension/extension/client.go b/apm-lambda-extension/extension/client.go index fd8d41d5..0c6c78b1 100644 --- a/apm-lambda-extension/extension/client.go +++ b/apm-lambda-extension/extension/client.go @@ -23,6 +23,7 @@ import ( "encoding/json" "fmt" "net/http" + "time" ) // RegisterResponse is the body of the response for /register @@ -34,6 +35,7 @@ type RegisterResponse struct { // NextEventResponse is the response for /event/next type NextEventResponse struct { + Timestamp time.Time `json:"timestamp,omitempty"` EventType EventType `json:"eventType"` DeadlineMs int64 `json:"deadlineMs"` RequestID string `json:"requestId"` diff --git a/apm-lambda-extension/extension/logger_test.go b/apm-lambda-extension/extension/logger_test.go index d869c808..05d5cedd 100644 --- a/apm-lambda-extension/extension/logger_test.go +++ b/apm-lambda-extension/extension/logger_test.go @@ -18,12 +18,13 @@ package extension import ( - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.uber.org/zap/zapcore" "io/ioutil" "os" "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap/zapcore" ) func init() { diff --git a/apm-lambda-extension/extension/process_metadata.go b/apm-lambda-extension/extension/process_metadata.go new file mode 100644 index 00000000..8ba5dca6 --- /dev/null +++ b/apm-lambda-extension/extension/process_metadata.go @@ -0,0 +1,78 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package extension + +import ( + "bufio" + "bytes" + "compress/gzip" + "compress/zlib" + "fmt" + "io/ioutil" + "strings" + + "github.com/pkg/errors" +) + +type MetadataContainer struct { + Metadata []byte +} + +// ProcessMetadata return a byte array containing the Metadata marshaled in JSON +// In case we want to update the Metadata values, usage of https://github.com/tidwall/sjson is advised +func ProcessMetadata(data AgentData) ([]byte, error) { + uncompressedData, err := GetUncompressedBytes(data.Data, data.ContentEncoding) + if err != nil { + return nil, errors.New(fmt.Sprintf("Error uncompressing agent data for metadata extraction : %v", err)) + } + scanner := bufio.NewScanner(strings.NewReader(string(uncompressedData))) + scanner.Scan() + if strings.Contains(strings.ToLower(scanner.Text()), "metadata") { + return scanner.Bytes(), nil + } + return nil, errors.New("No metadata found in APM agent payload") +} + +func GetUncompressedBytes(rawBytes []byte, encodingType string) ([]byte, error) { + switch encodingType { + case "deflate": + reader := bytes.NewReader([]byte(rawBytes)) + zlibreader, err := zlib.NewReader(reader) + if err != nil { + return nil, fmt.Errorf("could not create zlib.NewReader: %v", err) + } + bodyBytes, err := ioutil.ReadAll(zlibreader) + if err != nil { + return nil, fmt.Errorf("could not read from zlib reader using ioutil.ReadAll: %v", err) + } + return bodyBytes, nil + case "gzip": + reader := bytes.NewReader([]byte(rawBytes)) + zlibreader, err := gzip.NewReader(reader) + if err != nil { + return nil, fmt.Errorf("could not create gzip.NewReader: %v", err) + } + bodyBytes, err := ioutil.ReadAll(zlibreader) + if err != nil { + return nil, fmt.Errorf("could not read from gzip reader using ioutil.ReadAll: %v", err) + } + return bodyBytes, nil + default: + return rawBytes, nil + } +} diff --git a/apm-lambda-extension/extension/process_metadata_test.go b/apm-lambda-extension/extension/process_metadata_test.go new file mode 100644 index 00000000..f9ee03ad --- /dev/null +++ b/apm-lambda-extension/extension/process_metadata_test.go @@ -0,0 +1,49 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package extension + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func Test_processMetadata(t *testing.T) { + + // Copied from https://github.com/elastic/apm-server/blob/master/testdata/intake-v2/transactions.ndjson. + benchBody := []byte(`{"metadata": {"service": {"name": "1234_service-12a3","node": {"configured_name": "node-123"},"version": "5.1.3","environment": "staging","language": {"name": "ecmascript","version": "8"},"runtime": {"name": "node","version": "8.0.0"},"framework": {"name": "Express","version": "1.2.3"},"agent": {"name": "elastic-node","version": "3.14.0"}},"user": {"id": "123user", "username": "bar", "email": "bar@user.com"}, "labels": {"tag0": null, "tag1": "one", "tag2": 2}, "process": {"pid": 1234,"ppid": 6789,"title": "node","argv": ["node","server.js"]},"system": {"hostname": "prod1.example.com","architecture": "x64","platform": "darwin", "container": {"id": "container-id"}, "kubernetes": {"namespace": "namespace1", "pod": {"uid": "pod-uid", "name": "pod-name"}, "node": {"name": "node-name"}}},"cloud":{"account":{"id":"account_id","name":"account_name"},"availability_zone":"cloud_availability_zone","instance":{"id":"instance_id","name":"instance_name"},"machine":{"type":"machine_type"},"project":{"id":"project_id","name":"project_name"},"provider":"cloud_provider","region":"cloud_region","service":{"name":"lambda"}}}} +{"transaction": { "id": "945254c567a5417e", "trace_id": "0123456789abcdef0123456789abcdef", "parent_id": "abcdefabcdef01234567", "type": "request", "duration": 32.592981, "span_count": { "started": 43 }}} +{"transaction": {"id": "4340a8e0df1906ecbfa9", "trace_id": "0acd456789abcdef0123456789abcdef", "name": "GET /api/types","type": "request","duration": 32.592981,"outcome":"success", "result": "success", "timestamp": 1496170407154000, "sampled": true, "span_count": {"started": 17},"context": {"service": {"runtime": {"version": "7.0"}},"page":{"referer":"http://localhost:8000/test/e2e/","url":"http://localhost:8000/test/e2e/general-usecase/"}, "request": {"socket": {"remote_address": "12.53.12.1","encrypted": true},"http_version": "1.1","method": "POST","url": {"protocol": "https:","full": "https://www.example.com/p/a/t/h?query=string#hash","hostname": "www.example.com","port": "8080","pathname": "/p/a/t/h","search": "?query=string","hash": "#hash","raw": "/p/a/t/h?query=string#hash"},"headers": {"user-agent":["Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36","Mozilla Chrome Edge"],"content-type": "text/html","cookie": "c1=v1, c2=v2","some-other-header": "foo","array": ["foo","bar","baz"]},"cookies": {"c1": "v1","c2": "v2"},"env": {"SERVER_SOFTWARE": "nginx","GATEWAY_INTERFACE": "CGI/1.1"},"body": {"str": "hello world","additional": { "foo": {},"bar": 123,"req": "additional information"}}},"response": {"status_code": 200,"headers": {"content-type": "application/json"},"headers_sent": true,"finished": true,"transfer_size":25.8,"encoded_body_size":26.90,"decoded_body_size":29.90}, "user": {"domain": "ldap://abc","id": "99","username": "foo"},"tags": {"organization_uuid": "9f0e9d64-c185-4d21-a6f4-4673ed561ec8", "tag2": 12, "tag3": 12.45, "tag4": false, "tag5": null },"custom": {"my_key": 1,"some_other_value": "foo bar","and_objects": {"foo": ["bar","baz"]},"(": "not a valid regex and that is fine"}}}} +{"transaction": { "id": "cdef4340a8e0df19", "trace_id": "0acd456789abcdef0123456789abcdef", "type": "request", "duration": 13.980558, "timestamp": 1532976822281000, "sampled": null, "span_count": { "dropped": 55, "started": 436 }, "marks": {"navigationTiming": {"appBeforeBootstrap": 608.9300000000001,"navigationStart": -21},"another_mark": {"some_long": 10,"some_float": 10.0}, "performance": {}}, "context": { "request": { "socket": { "remote_address": "192.0.1", "encrypted": null }, "method": "POST", "headers": { "user-agent": null, "content-type": null, "cookie": null }, "url": { "protocol": null, "full": null, "hostname": null, "port": null, "pathname": null, "search": null, "hash": null, "raw": null } }, "response": { "headers": { "content-type": null } }, "service": {"environment":"testing","name": "service1","node": {"configured_name": "node-ABC"}, "language": {"version": "2.5", "name": "ruby"}, "agent": {"version": "2.2", "name": "elastic-ruby", "ephemeral_id": "justanid"}, "framework": {"version": "5.0", "name": "Rails"}, "version": "2", "runtime": {"version": "2.5", "name": "cruby"}}},"experience":{"cls":1,"fid":2.0,"tbt":3.4,"longtask":{"count":3,"sum":2.5,"max":1}}}} +{"transaction": { "id": "00xxxxFFaaaa1234", "trace_id": "0123456789abcdef0123456789abcdef", "name": "amqp receive", "parent_id": "abcdefabcdef01234567", "type": "messaging", "duration": 3, "span_count": { "started": 1 }, "context": {"message": {"queue": { "name": "new_users"}, "age":{ "ms": 1577958057123}, "headers": {"user_id": "1ax3", "involved_services": ["user", "auth"]}, "body": "user created", "routing_key": "user-created-transaction"}},"session":{"id":"sunday","sequence":123}}} +{"transaction": { "name": "july-2021-delete-after-july-31", "type": "lambda", "result": "success", "id": "142e61450efb8574", "trace_id": "eb56529a1f461c5e7e2f66ecb075e983", "subtype": null, "action": null, "duration": 38.853, "timestamp": 1631736666365048, "sampled": true, "context": { "cloud": { "origin": { "account": { "id": "abc123" }, "provider": "aws", "region": "us-east-1", "service": { "name": "serviceName" } } }, "service": { "origin": { "id": "abc123", "name": "service-name", "version": "1.0" } }, "user": {}, "tags": {}, "custom": { } }, "sync": true, "span_count": { "started": 0 }, "outcome": "unknown", "faas": { "coldstart": false, "execution": "2e13b309-23e1-417f-8bf7-074fc96bc683", "trigger": { "request_id": "FuH2Cir_vHcEMUA=", "type": "http" } }, "sample_rate": 1 } } +`) + + agentData := AgentData{Data: benchBody, ContentEncoding: ""} + extractedMetadata, err := ProcessMetadata(agentData) + require.NoError(t, err) + + // Metadata is extracted as is. + desiredMetadata := []byte(`{"metadata":{"service":{"name":"1234_service-12a3","version":"5.1.3","environment":"staging","agent":{"name":"elastic-node","version":"3.14.0"},"framework":{"name":"Express","version":"1.2.3"},"language":{"name":"ecmascript","version":"8"},"runtime":{"name":"node","version":"8.0.0"},"node":{"configured_name":"node-123"}},"user":{"username":"bar","id":"123user","email":"bar@user.com"},"labels":{"tag0":null,"tag1":"one","tag2":2},"process":{"pid":1234,"ppid":6789,"title":"node","argv":["node","server.js"]},"system":{"architecture":"x64","hostname":"prod1.example.com","platform":"darwin","container":{"id":"container-id"},"kubernetes":{"namespace":"namespace1","node":{"name":"node-name"},"pod":{"name":"pod-name","uid":"pod-uid"}}},"cloud":{"provider":"cloud_provider","region":"cloud_region","availability_zone":"cloud_availability_zone","instance":{"id":"instance_id","name":"instance_name"},"machine":{"type":"machine_type"},"account":{"id":"account_id","name":"account_name"},"project":{"id":"project_id","name":"project_name"},"service":{"name":"lambda"}}}}`) + if err != nil { + Log.Errorf("Could not marshal extracted Metadata : %v", err) + } + + assert.JSONEq(t, string(desiredMetadata), string(extractedMetadata)) +} diff --git a/apm-lambda-extension/extension/version.go b/apm-lambda-extension/extension/version.go new file mode 100644 index 00000000..9f7d806e --- /dev/null +++ b/apm-lambda-extension/extension/version.go @@ -0,0 +1,22 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package extension + +const ( + Version = "1.1.0" +) diff --git a/apm-lambda-extension/go.mod b/apm-lambda-extension/go.mod index cb532e06..24d99c06 100644 --- a/apm-lambda-extension/go.mod +++ b/apm-lambda-extension/go.mod @@ -16,10 +16,16 @@ require ( gotest.tools v2.2.0+incompatible ) +require ( + go.elastic.co/apm/v2 v2.1.0 + go.elastic.co/fastjson v1.1.0 +) + require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/google/go-cmp v0.5.6 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/santhosh-tekuri/jsonschema v1.2.4 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) diff --git a/apm-lambda-extension/go.sum b/apm-lambda-extension/go.sum index 4a908e66..71cb8653 100644 --- a/apm-lambda-extension/go.sum +++ b/apm-lambda-extension/go.sum @@ -1,3 +1,4 @@ +github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aws/aws-sdk-go v1.44.27 h1:8CMspeZSrewnbvAwgl8qo5R7orDLwQnTGBf/OKPiHxI= github.com/aws/aws-sdk-go v1.44.27/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= @@ -5,14 +6,22 @@ github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZx github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/elastic/go-licenser v0.4.0/go.mod h1:V56wHMpmdURfibNBggaSBfqgPxyT1Tldns1i87iTEvU= +github.com/elastic/go-sysinfo v1.7.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= +github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU= +github.com/elastic/go-windows v1.0.1/go.mod h1:FoVvqWSun28vaDQPbj2Elfc0JahhPB7WQEGa3c814Ss= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jcchavezs/porto v0.1.0/go.mod h1:fESH0gzDHiutHRdX2hv27ojnOVFco37hg1W6E9EZF4A= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg= github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= @@ -23,18 +32,31 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/magefile/mage v1.9.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magefile/mage v1.13.0 h1:XtLJl8bcCM7EFoO8FyH8XK3t7G5hQAeK+i4tq+veT9M= github.com/magefile/mage v1.13.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/procfs v0.0.0-20190425082905-87a4384529e0/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/santhosh-tekuri/jsonschema v1.2.4 h1:hNhW8e7t+H1vgY+1QeEQpveR6D4+OwKPXCfD2aieJis= +github.com/santhosh-tekuri/jsonschema v1.2.4/go.mod h1:TEAUOeZSmIxTTuHatJzrvARHiuO9LYd+cIxzgEHCQI4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.elastic.co/apm/v2 v2.1.0 h1:rkJSHE4ggekHhUR5v0KKkoMbrRSJN8YoBiEgQnkV1OY= +go.elastic.co/apm/v2 v2.1.0/go.mod h1:KGQn56LtRmkQjt2qw4+c1Jz8gv9rCBUU/m21uxrqcps= go.elastic.co/ecszap v1.0.1 h1:mBxqEJAEXBlpi5+scXdzL7LTFGogbuxipJC0KTZicyA= go.elastic.co/ecszap v1.0.1/go.mod h1:SVjazT+QgNeHSGOCUHvRgN+ZRj5FkB7IXQQsncdF57A= +go.elastic.co/fastjson v1.1.0 h1:3MrGBWWVIxe/xvsbpghtkFoPciPhOCmjsR/HfwEeQR4= +go.elastic.co/fastjson v1.1.0/go.mod h1:boNGISWMjQsUPy/t6yqt2/1Wx4YNPSe+mZjlyw9vKKI= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -48,30 +70,54 @@ go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191025021431-6c3a3bfe00ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211102192858-4dd72447c267/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200509030707-2212a7e161a5/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= +golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -80,6 +126,8 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -87,3 +135,5 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= +howett.net/plist v1.0.0/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g= diff --git a/apm-lambda-extension/logsapi/client.go b/apm-lambda-extension/logsapi/client.go index f8dd490c..093872c0 100644 --- a/apm-lambda-extension/logsapi/client.go +++ b/apm-lambda-extension/logsapi/client.go @@ -62,6 +62,8 @@ const ( // RuntimeDone event is sent when lambda function is finished it's execution RuntimeDone SubEventType = "platform.runtimeDone" Fault SubEventType = "platform.fault" + Report SubEventType = "platform.report" + Start SubEventType = "platform.start" ) // BufferingCfg is the configuration set for receiving logs from Logs API. Whichever of the conditions below is met first, the logs will be sent diff --git a/apm-lambda-extension/logsapi/process_metrics.go b/apm-lambda-extension/logsapi/process_metrics.go new file mode 100644 index 00000000..f857c85d --- /dev/null +++ b/apm-lambda-extension/logsapi/process_metrics.go @@ -0,0 +1,110 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package logsapi + +import ( + "context" + "math" + "strconv" + + "elastic/apm-lambda-extension/extension" + + "go.elastic.co/apm/v2/model" + "go.elastic.co/fastjson" +) + +type PlatformMetrics struct { + DurationMs float32 `json:"durationMs"` + BilledDurationMs int32 `json:"billedDurationMs"` + MemorySizeMB int32 `json:"memorySizeMB"` + MaxMemoryUsedMB int32 `json:"maxMemoryUsedMB"` + InitDurationMs float32 `json:"initDurationMs"` +} + +type MetricsContainer struct { + Metrics *model.Metrics `json:"metricset"` +} + +// Add adds a metric with the given name, labels, and value, +// The labels are expected to be sorted lexicographically. +func (mc MetricsContainer) Add(name string, value float64) { + mc.addMetric(name, model.Metric{Value: value}) +} + +// Simplified version of https://github.com/elastic/apm-agent-go/blob/675e8398c7fe546f9fd169bef971b9ccfbcdc71f/metrics.go#L89 +func (mc MetricsContainer) addMetric(name string, metric model.Metric) { + + if mc.Metrics.Samples == nil { + mc.Metrics.Samples = make(map[string]model.Metric) + } + mc.Metrics.Samples[name] = metric +} + +func (mc MetricsContainer) MarshalFastJSON(json *fastjson.Writer) error { + json.RawString(`{"metricset":`) + if err := mc.Metrics.MarshalFastJSON(json); err != nil { + return err + } + json.RawString(`}`) + return nil +} + +func ProcessPlatformReport(ctx context.Context, metadataContainer *extension.MetadataContainer, functionData *extension.NextEventResponse, platformReport LogEvent) (extension.AgentData, error) { + var metricsData []byte + metricsContainer := MetricsContainer{ + Metrics: &model.Metrics{}, + } + convMB2Bytes := float64(1024 * 1024) + platformReportMetrics := platformReport.Record.Metrics + + // APM Spec Fields + // Timestamp + metricsContainer.Metrics.Timestamp = model.Time(platformReport.Time) + // FaaS Fields + metricsContainer.Metrics.Labels = model.StringMap{ + {Key: "faas.execution", Value: platformReport.Record.RequestId}, + {Key: "faas.id", Value: functionData.InvokedFunctionArn}, + {Key: "faas.coldstart", Value: strconv.FormatBool(platformReportMetrics.InitDurationMs > 0)}, + } + // System + // AWS uses binary multiples to compute memory : https://aws.amazon.com/about-aws/whats-new/2020/12/aws-lambda-supports-10gb-memory-6-vcpu-cores-lambda-functions/ + metricsContainer.Add("system.memory.total", float64(platformReportMetrics.MemorySizeMB)*convMB2Bytes) // Unit : Bytes + metricsContainer.Add("system.memory.actual.free", float64(platformReportMetrics.MemorySizeMB-platformReportMetrics.MaxMemoryUsedMB)*convMB2Bytes) // Unit : Bytes + + // Raw Metrics + metricsContainer.Add("aws.lambda.metrics.duration", float64(platformReportMetrics.DurationMs)) // Unit : Milliseconds + metricsContainer.Add("aws.lambda.metrics.billed_duration", float64(platformReportMetrics.BilledDurationMs)) // Unit : Milliseconds + metricsContainer.Add("aws.lambda.metrics.coldstart_duration", float64(platformReportMetrics.InitDurationMs)) // Unit : Milliseconds + // In AWS Lambda, the Timeout is configured as an integer number of seconds. We use this assumption to derive the Timeout from + // - The epoch corresponding to the end of the current invocation (its "deadline") + // - The epoch corresponding to the start of the current invocation + // - The multiplication / division then rounds the value to obtain a number of ms that can be expressed a multiple of 1000 (see initial assumption) + metricsContainer.Add("aws.lambda.metrics.timeout", math.Ceil(float64(functionData.DeadlineMs-functionData.Timestamp.UnixMilli())/1e3)*1e3) // Unit : Milliseconds + + var jsonWriter fastjson.Writer + if err := metricsContainer.MarshalFastJSON(&jsonWriter); err != nil { + return extension.AgentData{Data: metricsData}, nil + } + + if metadataContainer.Metadata != nil { + metricsData = append(metadataContainer.Metadata, []byte("\n")...) + } + + metricsData = append(metricsData, jsonWriter.Bytes()...) + return extension.AgentData{Data: metricsData}, nil +} diff --git a/apm-lambda-extension/logsapi/process_metrics_test.go b/apm-lambda-extension/logsapi/process_metrics_test.go new file mode 100644 index 00000000..89d6649f --- /dev/null +++ b/apm-lambda-extension/logsapi/process_metrics_test.go @@ -0,0 +1,92 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package logsapi + +import ( + "context" + "fmt" + "log" + "strings" + "testing" + "time" + + "elastic/apm-lambda-extension/extension" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func Test_processPlatformReport(t *testing.T) { + + mc := extension.MetadataContainer{ + Metadata: []byte(fmt.Sprintf(`{"metadata":{"service":{"agent":{"name":"apm-lambda-extension","version":"%s"},"framework":{"name":"AWS Lambda","version":""},"language":{"name":"python","version":"3.9.8"},"runtime":{"name":"","version":""},"node":{}},"user":{},"process":{"pid":0},"system":{"container":{"id":""},"kubernetes":{"node":{},"pod":{}}},"cloud":{"provider":"","instance":{},"machine":{},"account":{},"project":{},"service":{}}}}`, extension.Version)), + } + + timestamp := time.Now() + + pm := PlatformMetrics{ + DurationMs: 182.43, + BilledDurationMs: 183, + MemorySizeMB: 128, + MaxMemoryUsedMB: 76, + InitDurationMs: 422.97, + } + + logEventRecord := LogEventRecord{ + RequestId: "6f7f0961f83442118a7af6fe80b88d56", + Status: "Available", + Metrics: pm, + } + + logEvent := LogEvent{ + Time: timestamp, + Type: "platform.report", + StringRecord: "", + Record: logEventRecord, + } + + event := extension.NextEventResponse{ + Timestamp: timestamp, + EventType: extension.Invoke, + DeadlineMs: timestamp.UnixNano()/1e6 + 4584, // Milliseconds + RequestID: "8476a536-e9f4-11e8-9739-2dfe598c3fcd", + InvokedFunctionArn: "arn:aws:lambda:us-east-2:123456789012:function:custom-runtime", + Tracing: extension.Tracing{ + Type: "None", + Value: "None", + }, + } + + desiredOutputMetadata := fmt.Sprintf(`{"metadata":{"service":{"agent":{"name":"apm-lambda-extension","version":"%s"},"framework":{"name":"AWS Lambda","version":""},"language":{"name":"python","version":"3.9.8"},"runtime":{"name":"","version":""},"node":{}},"user":{},"process":{"pid":0},"system":{"container":{"id":""},"kubernetes":{"node":{},"pod":{}}},"cloud":{"provider":"","instance":{},"machine":{},"account":{},"project":{},"service":{}}}}`, extension.Version) + + desiredOutputMetrics := fmt.Sprintf(`{"metricset":{"samples":{"aws.lambda.metrics.billed_duration":{"value":183},"aws.lambda.metrics.coldstart_duration":{"value":422.9700012207031},"aws.lambda.metrics.timeout":{"value":5000},"system.memory.total":{"value":1.34217728e+08},"system.memory.actual.free":{"value":5.4525952e+07},"aws.lambda.metrics.duration":{"value":182.42999267578125}},"timestamp":%d,"tags":{"faas.execution":"6f7f0961f83442118a7af6fe80b88d56","faas.id":"arn:aws:lambda:us-east-2:123456789012:function:custom-runtime","faas.coldstart":"true"}}}`, timestamp.UnixNano()/1e3) + + rawBytes, err := ProcessPlatformReport(context.Background(), &mc, &event, logEvent) + require.NoError(t, err) + + requestBytes, err := extension.GetUncompressedBytes(rawBytes.Data, "") + require.NoError(t, err) + + out := string(requestBytes) + log.Println(out) + + processingResult := strings.Split(string(requestBytes), "\n") + + assert.JSONEq(t, desiredOutputMetadata, processingResult[0]) + assert.JSONEq(t, desiredOutputMetrics, processingResult[1]) +} diff --git a/apm-lambda-extension/logsapi/route_handlers.go b/apm-lambda-extension/logsapi/route_handlers.go index 3da4c316..7606f654 100644 --- a/apm-lambda-extension/logsapi/route_handlers.go +++ b/apm-lambda-extension/logsapi/route_handlers.go @@ -18,10 +18,11 @@ package logsapi import ( - "elastic/apm-lambda-extension/extension" "encoding/json" "net/http" "time" + + "elastic/apm-lambda-extension/extension" ) func handleLogEventsRequest(transport *LogsTransport) func(w http.ResponseWriter, r *http.Request) { diff --git a/apm-lambda-extension/logsapi/route_handlers_test.go b/apm-lambda-extension/logsapi/route_handlers_test.go index 9bb2f6e3..f6fde29d 100644 --- a/apm-lambda-extension/logsapi/route_handlers_test.go +++ b/apm-lambda-extension/logsapi/route_handlers_test.go @@ -48,6 +48,13 @@ func TestLogEventUnmarshalReport(t *testing.T) { rec := LogEventRecord{ RequestId: "6f7f0961f83442118a7af6fe80b88d56", Status: "", // no status was given in sample json + Metrics: PlatformMetrics{ + DurationMs: 101.51, + BilledDurationMs: 300, + MemorySizeMB: 512, + MaxMemoryUsedMB: 33, + InitDurationMs: 116.67, + }, } assert.Equal(t, rec, le.Record) diff --git a/apm-lambda-extension/logsapi/subscribe.go b/apm-lambda-extension/logsapi/subscribe.go index 3d844bd3..d93ba0f5 100644 --- a/apm-lambda-extension/logsapi/subscribe.go +++ b/apm-lambda-extension/logsapi/subscribe.go @@ -19,13 +19,14 @@ package logsapi import ( "context" - "elastic/apm-lambda-extension/extension" "fmt" "net" "net/http" "os" "time" + "elastic/apm-lambda-extension/extension" + "github.com/pkg/errors" ) @@ -57,8 +58,9 @@ type LogEvent struct { // LogEventRecord is a sub-object in a Logs API event type LogEventRecord struct { - RequestId string `json:"requestId"` - Status string `json:"status"` + RequestId string `json:"requestId"` + Status string `json:"status"` + Metrics PlatformMetrics `json:"metrics"` } // Subscribes to the Logs API @@ -156,16 +158,25 @@ func checkLambdaFunction() bool { return false } -// WaitRuntimeDone consumes events until a RuntimeDone event corresponding +// ProcessLogs consumes events until a RuntimeDone event corresponding // to requestID is received, or ctx is cancelled, and then returns. -func WaitRuntimeDone(ctx context.Context, requestID string, transport *LogsTransport, runtimeDoneSignal chan struct{}) error { +func ProcessLogs( + ctx context.Context, + requestID string, + apmServerTransport *extension.ApmServerTransport, + logsTransport *LogsTransport, + metadataContainer *extension.MetadataContainer, + runtimeDoneSignal chan struct{}, + prevEvent *extension.NextEventResponse, +) error { for { select { - case logEvent := <-transport.logsChannel: + case logEvent := <-logsTransport.logsChannel: extension.Log.Debugf("Received log event %v", logEvent.Type) + switch logEvent.Type { // Check the logEvent for runtimeDone and compare the RequestID // to the id that came in via the Next API - if logEvent.Type == RuntimeDone { + case RuntimeDone: if logEvent.Record.RequestId == requestID { extension.Log.Info("Received runtimeDone event for this function invocation") runtimeDoneSignal <- struct{}{} @@ -173,6 +184,20 @@ func WaitRuntimeDone(ctx context.Context, requestID string, transport *LogsTrans } else { extension.Log.Debug("Log API runtimeDone event request id didn't match") } + // Check if the logEvent contains metrics and verify that they can be linked to the previous invocation + case Report: + if prevEvent != nil && logEvent.Record.RequestId == prevEvent.RequestID { + extension.Log.Debug("Received platform report for the previous function invocation") + processedMetrics, err := ProcessPlatformReport(ctx, metadataContainer, prevEvent, logEvent) + if err != nil { + extension.Log.Errorf("Error processing Lambda platform metrics : %v", err) + } else { + apmServerTransport.EnqueueAPMData(processedMetrics) + } + } else { + extension.Log.Warn("report event request id didn't match the previous event id") + extension.Log.Debug("Log API runtimeDone event request id didn't match") + } } case <-ctx.Done(): extension.Log.Debug("Current invocation over. Interrupting logs processing goroutine") diff --git a/apm-lambda-extension/main.go b/apm-lambda-extension/main.go index ae16b15f..5ebc2238 100644 --- a/apm-lambda-extension/main.go +++ b/apm-lambda-extension/main.go @@ -97,24 +97,40 @@ func main() { extension.Log.Warnf("Error while subscribing to the Logs API: %v", err) } + // The previous event id is used to validate the received Lambda metrics + var prevEvent *extension.NextEventResponse + // This data structure contains metadata tied to the current Lambda instance. If empty, it is populated once for each + // active Lambda environment + metadataContainer := extension.MetadataContainer{} + for { select { case <-ctx.Done(): return default: var backgroundDataSendWg sync.WaitGroup - processEvent(ctx, cancel, apmServerTransport, logsTransport, &backgroundDataSendWg) + event := processEvent(ctx, cancel, apmServerTransport, logsTransport, &backgroundDataSendWg, prevEvent, &metadataContainer) extension.Log.Debug("Waiting for background data send to end") backgroundDataSendWg.Wait() if config.SendStrategy == extension.SyncFlush { // Flush APM data now that the function invocation has completed apmServerTransport.FlushAPMData(ctx) } + prevEvent = event } } } -func processEvent(ctx context.Context, cancel context.CancelFunc, apmServerTransport *extension.ApmServerTransport, logsTransport *logsapi.LogsTransport, backgroundDataSendWg *sync.WaitGroup) { +func processEvent( + ctx context.Context, + cancel context.CancelFunc, + apmServerTransport *extension.ApmServerTransport, + logsTransport *logsapi.LogsTransport, + backgroundDataSendWg *sync.WaitGroup, + prevEvent *extension.NextEventResponse, + metadataContainer *extension.MetadataContainer, +) *extension.NextEventResponse { + // Invocation context invocationCtx, invocationCancel := context.WithCancel(ctx) defer invocationCancel() @@ -131,15 +147,17 @@ func processEvent(ctx context.Context, cancel context.CancelFunc, apmServerTrans extension.Log.Errorf("Error: %s", err) extension.Log.Infof("Exit signal sent to runtime : %s", status) extension.Log.Infof("Exiting") - return + return nil } + // Used to compute Lambda Timeout + event.Timestamp = time.Now() extension.Log.Debug("Received event.") extension.Log.Debugf("%v", extension.PrettyPrint(event)) if event.EventType == extension.Shutdown { cancel() - return + return event } // APM Data Processing @@ -148,17 +166,17 @@ func processEvent(ctx context.Context, cancel context.CancelFunc, apmServerTrans backgroundDataSendWg.Add(1) go func() { defer backgroundDataSendWg.Done() - if err := apmServerTransport.ForwardApmData(invocationCtx); err != nil { + if err := apmServerTransport.ForwardApmData(invocationCtx, metadataContainer); err != nil { extension.Log.Error(err) } }() - // Lambda Service Logs Processing + // Lambda Service Logs Processing, also used to extract metrics from APM logs // This goroutine should not be started if subscription failed runtimeDone := make(chan struct{}) if logsTransport != nil { go func() { - if err := logsapi.WaitRuntimeDone(invocationCtx, event.RequestID, logsTransport, runtimeDone); err != nil { + if err := logsapi.ProcessLogs(invocationCtx, event.RequestID, apmServerTransport, logsTransport, metadataContainer, runtimeDone, prevEvent); err != nil { extension.Log.Errorf("Error while processing Lambda Logs ; %v", err) } else { close(runtimeDone) @@ -191,4 +209,6 @@ func processEvent(ctx context.Context, cancel context.CancelFunc, apmServerTrans case <-timer.C: extension.Log.Info("Time expired waiting for agent signal or runtimeDone event") } + + return event } diff --git a/apm-lambda-extension/main_test.go b/apm-lambda-extension/main_test.go index 28e66d26..9c56ed6b 100644 --- a/apm-lambda-extension/main_test.go +++ b/apm-lambda-extension/main_test.go @@ -46,7 +46,8 @@ const ( InvokeHang MockEventType = "Hang" InvokeStandard MockEventType = "Standard" InvokeStandardInfo MockEventType = "StandardInfo" - InvokeStandardFlush MockEventType = "Flush" + InvokeStandardFlush MockEventType = "StandardFlush" + InvokeStandardMetadata MockEventType = "StandardMetadata" InvokeLateFlush MockEventType = "LateFlush" InvokeWaitgroupsRace MockEventType = "InvokeWaitgroupsRace" InvokeMultipleTransactionsOverload MockEventType = "MultipleTransactionsOverload" @@ -111,8 +112,6 @@ func newMockApmServer(t *testing.T) (*MockServerInternals, *httptest.Server) { case Crashes: panic("Server crashed") default: - w.WriteHeader(http.StatusInternalServerError) - return } if r.RequestURI == "/intake/v2/events" { w.WriteHeader(http.StatusAccepted) @@ -211,9 +210,10 @@ func newTestStructs(t *testing.T) chan MockEvent { } func processMockEvent(currId string, event MockEvent, extensionPort string, internals *MockServerInternals) { - sendLogEvent(currId, "platform.start") + sendLogEvent(currId, logsapi.Start) client := http.Client{} sendRuntimeDone := true + sendMetrics := true switch event.Type { case InvokeHang: time.Sleep(time.Duration(event.Timeout) * time.Second) @@ -222,6 +222,12 @@ func processMockEvent(currId string, event MockEvent, extensionPort string, inte req, _ := http.NewRequest("POST", fmt.Sprintf("http://localhost:%s/intake/v2/events", extensionPort), bytes.NewBuffer([]byte(event.APMServerBehavior))) res, _ := client.Do(req) extension.Log.Debugf("Response seen by the agent : %d", res.StatusCode) + case InvokeStandardMetadata: + time.Sleep(time.Duration(event.ExecutionDuration) * time.Second) + metadata := `{"metadata":{"service":{"name":"1234_service-12a3","version":"5.1.3","environment":"staging","agent":{"name":"elastic-node","version":"3.14.0"},"framework":{"name":"Express","version":"1.2.3"},"language":{"name":"ecmascript","version":"8"},"runtime":{"name":"node","version":"8.0.0"},"node":{"configured_name":"node-123"}},"user":{"username":"bar","id":"123user","email":"bar@user.com"},"labels":{"tag0":null,"tag1":"one","tag2":2},"process":{"pid":1234,"ppid":6789,"title":"node","argv":["node","server.js"]},"system":{"architecture":"x64","hostname":"prod1.example.com","platform":"darwin","container":{"id":"container-id"},"kubernetes":{"namespace":"namespace1","node":{"name":"node-name"},"pod":{"name":"pod-name","uid":"pod-uid"}}},"cloud":{"provider":"cloud_provider","region":"cloud_region","availability_zone":"cloud_availability_zone","instance":{"id":"instance_id","name":"instance_name"},"machine":{"type":"machine_type"},"account":{"id":"account_id","name":"account_name"},"project":{"id":"project_id","name":"project_name"},"service":{"name":"lambda"}}}}` + req, _ := http.NewRequest("POST", fmt.Sprintf("http://localhost:%s/intake/v2/events", extensionPort), bytes.NewBuffer([]byte(metadata))) + res, _ := client.Do(req) + extension.Log.Debugf("Response seen by the agent : %d", res.StatusCode) case InvokeStandardFlush: time.Sleep(time.Duration(event.ExecutionDuration) * time.Second) reqData, _ := http.NewRequest("POST", fmt.Sprintf("http://localhost:%s/intake/v2/events?flushed=true", extensionPort), bytes.NewBuffer([]byte(event.APMServerBehavior))) @@ -238,6 +244,8 @@ func processMockEvent(currId string, event MockEvent, extensionPort string, inte } internals.WaitGroup.Done() }() + // For this specific scenario, we do not want to see metrics in the APM Server logs (in order to easily check if the logs contain to "TimelyResponse" back to back). + sendMetrics = false case InvokeWaitgroupsRace: time.Sleep(time.Duration(event.ExecutionDuration) * time.Second) reqData0, _ := http.NewRequest("POST", fmt.Sprintf("http://localhost:%s/intake/v2/events", extensionPort), bytes.NewBuffer([]byte(event.APMServerBehavior))) @@ -280,7 +288,10 @@ func processMockEvent(currId string, event MockEvent, extensionPort string, inte case Shutdown: } if sendRuntimeDone { - sendLogEvent(currId, "platform.runtimeDone") + sendLogEvent(currId, logsapi.RuntimeDone) + } + if sendMetrics { + sendLogEvent(currId, logsapi.Report) } } @@ -305,6 +316,16 @@ func sendLogEvent(requestId string, logEventType logsapi.SubEventType) { record := logsapi.LogEventRecord{ RequestId: requestId, } + if logEventType == logsapi.Report { + record.Metrics = logsapi.PlatformMetrics{ + BilledDurationMs: 60, + DurationMs: 59.9, + MemorySizeMB: 128, + MaxMemoryUsedMB: 60, + InitDurationMs: 500, + } + } + logEvent := logsapi.LogEvent{ Time: time.Now(), Type: logEventType, @@ -559,3 +580,44 @@ func TestInfoRequestHangs(t *testing.T) { assert.NotContains(t, lambdaServerInternals.Data, "7814d524d3602e70b703539c57568cba6964fc20") apmServerInternals.UnlockSignalChannel <- struct{}{} } + +// TestMetricsWithoutMetadata checks if the extension sends metrics corresponding to invocation n during invocation +// n+1, even if the metadata container was not populated +func TestMetricsWithoutMetadata(t *testing.T) { + initLogLevel(t, "trace") + eventsChannel := newTestStructs(t) + apmServerInternals, _ := newMockApmServer(t) + newMockLambdaServer(t, eventsChannel) + + eventsChain := []MockEvent{ + {Type: InvokeStandard, APMServerBehavior: TimelyResponse, ExecutionDuration: 1, Timeout: 5}, + {Type: InvokeStandard, APMServerBehavior: TimelyResponse, ExecutionDuration: 1, Timeout: 5}, + } + eventQueueGenerator(eventsChain, eventsChannel) + assert.NotPanics(t, main) + + assert.Contains(t, apmServerInternals.Data, `aws.lambda.metrics.billed_duration":{"value":60`) + assert.Contains(t, apmServerInternals.Data, `aws.lambda.metrics.duration":{"value":59.9`) + assert.Contains(t, apmServerInternals.Data, `aws.lambda.metrics.coldstart_duration":{"value":500`) +} + +// TestMetricsWithMetadata checks if the extension sends metrics corresponding to invocation n during invocation +// n+1, even if the metadata container was not populated +func TestMetricsWithMetadata(t *testing.T) { + initLogLevel(t, "trace") + eventsChannel := newTestStructs(t) + apmServerInternals, _ := newMockApmServer(t) + newMockLambdaServer(t, eventsChannel) + + eventsChain := []MockEvent{ + {Type: InvokeStandardMetadata, APMServerBehavior: TimelyResponse, ExecutionDuration: 1, Timeout: 5}, + {Type: InvokeStandardMetadata, APMServerBehavior: TimelyResponse, ExecutionDuration: 1, Timeout: 5}, + } + eventQueueGenerator(eventsChain, eventsChannel) + assert.NotPanics(t, main) + + assert.Contains(t, apmServerInternals.Data, `{"metadata":{"service":{"name":"1234_service-12a3","version":"5.1.3","environment":"staging","agent":{"name":"elastic-node","version":"3.14.0"},"framework":{"name":"Express","version":"1.2.3"},"language":{"name":"ecmascript","version":"8"},"runtime":{"name":"node","version":"8.0.0"},"node":{"configured_name":"node-123"}},"user":{"username":"bar","id":"123user","email":"bar@user.com"},"labels":{"tag0":null,"tag1":"one","tag2":2},"process":{"pid":1234,"ppid":6789,"title":"node","argv":["node","server.js"]},"system":{"architecture":"x64","hostname":"prod1.example.com","platform":"darwin","container":{"id":"container-id"},"kubernetes":{"namespace":"namespace1","node":{"name":"node-name"},"pod":{"name":"pod-name","uid":"pod-uid"}}},"cloud":{"provider":"cloud_provider","region":"cloud_region","availability_zone":"cloud_availability_zone","instance":{"id":"instance_id","name":"instance_name"},"machine":{"type":"machine_type"},"account":{"id":"account_id","name":"account_name"},"project":{"id":"project_id","name":"project_name"},"service":{"name":"lambda"}}}}`) + assert.Contains(t, apmServerInternals.Data, `aws.lambda.metrics.billed_duration":{"value":60`) + assert.Contains(t, apmServerInternals.Data, `aws.lambda.metrics.duration":{"value":59.9`) + assert.Contains(t, apmServerInternals.Data, `aws.lambda.metrics.coldstart_duration":{"value":500`) +}