From 944e71ffca910c8951dfa07b1b507520407edf53 Mon Sep 17 00:00:00 2001 From: Jean-Louis Voiseux <48380853+jlvoiseux@users.noreply.github.com> Date: Tue, 31 May 2022 18:02:13 +0200 Subject: [PATCH 01/21] Add model package for metrics This commit is heavily inspired by the Go agent codebase. The data model needed to (un)marshal metrics as well as the associated metadata is defined. --- apm-lambda-extension/model/apm_common.go | 20 ++ apm-lambda-extension/model/apm_metadata.go | 245 +++++++++++++++++++++ apm-lambda-extension/model/apm_metrics.go | 75 +++++++ 3 files changed, 340 insertions(+) create mode 100644 apm-lambda-extension/model/apm_common.go create mode 100644 apm-lambda-extension/model/apm_metadata.go create mode 100644 apm-lambda-extension/model/apm_metrics.go diff --git a/apm-lambda-extension/model/apm_common.go b/apm-lambda-extension/model/apm_common.go new file mode 100644 index 00000000..35f3cc46 --- /dev/null +++ b/apm-lambda-extension/model/apm_common.go @@ -0,0 +1,20 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package model + +type StringMap map[string]interface{} diff --git a/apm-lambda-extension/model/apm_metadata.go b/apm-lambda-extension/model/apm_metadata.go new file mode 100644 index 00000000..2f12f5ef --- /dev/null +++ b/apm-lambda-extension/model/apm_metadata.go @@ -0,0 +1,245 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package model + +// Service represents the service handling transactions being traced. +type Service struct { + // Name is the immutable name of the service. + Name string `json:"name,omitempty"` + + // Version is the version of the service, if it has one. + Version string `json:"version,omitempty"` + + // Environment is the name of the service's environment, if it has + // one, e.g. "production" or "staging". + Environment string `json:"environment,omitempty"` + + // Agent holds information about the Elastic APM agent tracing this + // service's transactions. + Agent Agent `json:"agent,omitempty"` + + // Framework holds information about the service's framework, if any. + Framework Framework `json:"framework,omitempty"` + + // Language holds information about the programming language in which + // the service is written. + Language Language `json:"language,omitempty"` + + // Runtime holds information about the programming language runtime + // running this service. + Runtime Runtime `json:"runtime,omitempty"` + + // Node holds unique information about each service node + Node ServiceNode `json:"node,omitempty"` +} + +// Agent holds information about the Elastic APM agent. +type Agent struct { + // Name is the name of the Elastic APM agent, e.g. "Go". + Name string `json:"name"` + + // Version is the version of the Elastic APM agent, e.g. "1.0.0". + Version string `json:"version"` +} + +// Framework holds information about the framework (typically web) +// used by the service. +type Framework struct { + // Name is the name of the framework. + Name string `json:"name"` + + // Version is the version of the framework. + Version string `json:"version"` +} + +// Language holds information about the programming language used. +type Language struct { + // Name is the name of the programming language. + Name string `json:"name"` + + // Version is the version of the programming language. + Version string `json:"version,omitempty"` +} + +// Runtime holds information about the programming language runtime. +type Runtime struct { + // Name is the name of the programming language runtime. + Name string `json:"name"` + + // Version is the version of the programming language runtime. + Version string `json:"version"` +} + +// ServiceNode holds unique information about each service node +type ServiceNode struct { + // ConfiguredName holds the name of the service node + ConfiguredName string `json:"configured_name,omitempty"` +} + +// System represents the system (operating system and machine) running the +// service. +type System struct { + // Architecture is the system's hardware architecture. + Architecture string `json:"architecture,omitempty"` + + // Hostname is the system's hostname. + Hostname string `json:"hostname,omitempty"` + + // Platform is the system's platform, or operating system name. + Platform string `json:"platform,omitempty"` + + // Container describes the container running the service. + Container Container `json:"container,omitempty"` + + // Kubernetes describes the kubernetes node and pod running the service. + Kubernetes Kubernetes `json:"kubernetes,omitempty"` +} + +// Process represents an operating system process. +type Process struct { + // Pid is the process ID. + Pid int `json:"pid"` + + // Ppid is the parent process ID, if known. + Ppid *int `json:"ppid,omitempty"` + + // Title is the title of the process. + Title string `json:"title,omitempty"` + + // Argv holds the command line arguments used to start the process. + Argv []string `json:"argv,omitempty"` +} + +// Container represents the container (e.g. Docker) running the service. +type Container struct { + // ID is the unique container ID. + ID string `json:"id"` +} + +// Kubernetes describes properties of the Kubernetes node and pod in which +// the service is running. +type Kubernetes struct { + // Namespace names the Kubernetes namespace in which the pod exists. + Namespace string `json:"namespace,omitempty"` + + // Node describes the Kubernetes node running the service's pod. + Node KubernetesNode `json:"node,omitempty"` + + // Pod describes the Kubernetes pod running the service. + Pod KubernetesPod `json:"pod,omitempty"` +} + +// KubernetesNode describes a Kubernetes node. +type KubernetesNode struct { + // Name holds the node name. + Name string `json:"name,omitempty"` +} + +// KubernetesPod describes a Kubernetes pod. +type KubernetesPod struct { + // Name holds the pod name. + Name string `json:"name,omitempty"` + + // UID holds the pod UID. + UID string `json:"uid,omitempty"` +} + +// Cloud represents the cloud in which the service is running. +type Cloud struct { + // Provider is the cloud provider name, e.g. aws, azure, gcp. + Provider string `json:"provider"` + + // Region is the cloud region name, e.g. us-east-1. + Region string `json:"region,omitempty"` + + // AvailabilityZone is the cloud availability zone name, e.g. us-east-1a. + AvailabilityZone string `json:"availability_zone,omitempty"` + + // Instance holds information about the cloud instance (virtual machine). + Instance CloudInstance `json:"instance,omitempty"` + + // Machine also holds information about the cloud instance (virtual machine). + Machine CloudMachine `json:"machine,omitempty"` + + // Account holds information about the cloud account. + Account CloudAccount `json:"account,omitempty"` + + // Project holds information about the cloud project. + Project CloudProject `json:"project,omitempty"` + + Service CloudService `json:"service,omitempty"` +} + +// CloudInstance holds information about a cloud instance (virtual machine). +type CloudInstance struct { + // ID holds the cloud instance identifier. + ID string `json:"id,omitempty"` + + // ID holds the cloud instance name. + Name string `json:"name,omitempty"` +} + +// CloudMachine holds information about a cloud instance (virtual machine). +type CloudMachine struct { + // Type holds the cloud instance type, e.g. t2.medium. + Type string `json:"type,omitempty"` +} + +// CloudAccount holds information about a cloud account. +type CloudAccount struct { + // ID holds the cloud account identifier. + ID string `json:"id,omitempty"` + + // ID holds the cloud account name. + Name string `json:"name,omitempty"` +} + +// CloudProject holds information about a cloud project. +type CloudProject struct { + // ID holds the cloud project identifier. + ID string `json:"id,omitempty"` + + // Name holds the cloud project name. + Name string `json:"name,omitempty"` +} + +type CloudService struct { + Name string `json:"name,omitempty"` +} + +// User holds information about an authenticated user. +type User struct { + // Username holds the username of the user. + Username string `json:"username,omitempty"` + + // ID identifies the user, e.g. a primary key. This may be + // a string or number. + ID string `json:"id,omitempty"` + + // Email holds the email address of the user. + Email string `json:"email,omitempty"` +} + +type Metadata struct { + Service Service `json:"service,omitempty"` + User User `json:"user,omitempty"` + Labels map[string]interface{} `json:"labels,omitempty"` + Process Process `json:"process,omitempty"` + System System `json:"system,omitempty"` + Cloud Cloud `json:"cloud,omitempty"` +} diff --git a/apm-lambda-extension/model/apm_metrics.go b/apm-lambda-extension/model/apm_metrics.go new file mode 100644 index 00000000..8b8b7a62 --- /dev/null +++ b/apm-lambda-extension/model/apm_metrics.go @@ -0,0 +1,75 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package model + +// MetricLabel is a name/value pair for labeling metrics. +type MetricLabel struct { + // Name is the label name. + Name string + + // Value is the label value. + Value string +} + +// MetricsTransaction holds transaction identifiers for metrics. +type MetricsTransaction struct { + Type string `json:"type,omitempty"` + Name string `json:"name,omitempty"` +} + +// MetricsSpan holds span identifiers for metrics. +type MetricsSpan struct { + Type string `json:"type,omitempty"` + Subtype string `json:"subtype,omitempty"` +} + +// Metric holds metric values. +type Metric struct { + Type string `json:"type,omitempty"` + // Value holds the metric value. + Value float64 `json:"value"` + // Buckets holds the metric bucket values. + Values []float64 `json:"values,omitempty"` + // Count holds the metric observation count for the bucket. + Counts []uint64 `json:"counts,omitempty"` +} + +// Metrics holds a set of metric samples, with an optional set of labels. +type Metrics struct { + // Timestamp holds the time at which the metric samples were taken. + Timestamp int64 `json:"timestamp"` + + // Transaction optionally holds the name and type of transactions + // with which these metrics are associated. + Transaction MetricsTransaction `json:"transaction,omitempty"` + + // Span optionally holds the type and subtype of the spans with + // which these metrics are associated. + Span MetricsSpan `json:"span,omitempty"` + + // Labels holds a set of labels associated with the metrics. + // The labels apply uniformly to all metric samples in the set. + // + // NOTE(axw) the schema calls the field "tags", but we use + // "labels" for agent-internal consistency. Labels aligns better + // with the common schema, anyway. + Labels StringMap `json:"tags,omitempty"` + + // Samples holds a map of metric samples, keyed by metric name. + Samples map[string]Metric `json:"samples"` +} From 1ec30ab1bbf383949b8084bfa5a1ae0089f7747d Mon Sep 17 00:00:00 2001 From: Jean-Louis Voiseux <48380853+jlvoiseux@users.noreply.github.com> Date: Tue, 31 May 2022 18:08:03 +0200 Subject: [PATCH 02/21] Add version utility file This file contains information about the extension version. This used to enrich Metrics metadata with the version of the extension used to generate these metrics. --- apm-lambda-extension/extension/version.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 apm-lambda-extension/extension/version.go diff --git a/apm-lambda-extension/extension/version.go b/apm-lambda-extension/extension/version.go new file mode 100644 index 00000000..9f7d806e --- /dev/null +++ b/apm-lambda-extension/extension/version.go @@ -0,0 +1,22 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package extension + +const ( + Version = "1.1.0" +) From 10e2c77dc8b3af3325fc58058be22c86c1d7ff66 Mon Sep 17 00:00:00 2001 From: Jean-Louis Voiseux <48380853+jlvoiseux@users.noreply.github.com> Date: Tue, 31 May 2022 18:20:07 +0200 Subject: [PATCH 03/21] Add metadata extraction from the first received transaction This commit allows for the extraction of the metadata from the first received transaction, with the assumption that this metadata will persist across invocations between cold starts. Metadata is obtained from the uncompressed payload, and a dedicated container is populated. --- apm-lambda-extension/e2e-testing/e2e_util.go | 34 +------ .../extension/process_metadata.go | 88 +++++++++++++++++++ .../extension/process_metadata_test.go | 70 +++++++++++++++ 3 files changed, 161 insertions(+), 31 deletions(-) create mode 100644 apm-lambda-extension/extension/process_metadata.go create mode 100644 apm-lambda-extension/extension/process_metadata_test.go diff --git a/apm-lambda-extension/e2e-testing/e2e_util.go b/apm-lambda-extension/e2e-testing/e2e_util.go index 485c097a..c88f3089 100644 --- a/apm-lambda-extension/e2e-testing/e2e_util.go +++ b/apm-lambda-extension/e2e-testing/e2e_util.go @@ -20,10 +20,6 @@ package e2eTesting import ( "archive/zip" "bufio" - "bytes" - "compress/gzip" - "compress/zlib" - "elastic/apm-lambda-extension/extension" "fmt" "io" "io/ioutil" @@ -33,6 +29,8 @@ import ( "os/exec" "path/filepath" "strings" + + "elastic/apm-lambda-extension/extension" ) // GetEnvVarValueOrSetDefault retrieves the environment variable envVarName. @@ -157,33 +155,7 @@ func GetDecompressedBytesFromRequest(req *http.Request) ([]byte, error) { if req.Body != nil { rawBytes, _ = ioutil.ReadAll(req.Body) } - - switch req.Header.Get("Content-Encoding") { - case "deflate": - reader := bytes.NewReader(rawBytes) - zlibreader, err := zlib.NewReader(reader) - if err != nil { - return nil, fmt.Errorf("could not create zlib.NewReader: %v", err) - } - bodyBytes, err := ioutil.ReadAll(zlibreader) - if err != nil { - return nil, fmt.Errorf("could not read from zlib reader using ioutil.ReadAll: %v", err) - } - return bodyBytes, nil - case "gzip": - reader := bytes.NewReader(rawBytes) - zlibreader, err := gzip.NewReader(reader) - if err != nil { - return nil, fmt.Errorf("could not create gzip.NewReader: %v", err) - } - bodyBytes, err := ioutil.ReadAll(zlibreader) - if err != nil { - return nil, fmt.Errorf("could not read from gzip reader using ioutil.ReadAll: %v", err) - } - return bodyBytes, nil - default: - return rawBytes, nil - } + return extension.GetUncompressedBytes(rawBytes, req.Header.Get("Content-Encoding")) } // GetFreePort is a function that queries the kernel and obtains an unused port. diff --git a/apm-lambda-extension/extension/process_metadata.go b/apm-lambda-extension/extension/process_metadata.go new file mode 100644 index 00000000..27f757a4 --- /dev/null +++ b/apm-lambda-extension/extension/process_metadata.go @@ -0,0 +1,88 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package extension + +import ( + "bytes" + "compress/gzip" + "compress/zlib" + "encoding/json" + "fmt" + "io" + "io/ioutil" + + "elastic/apm-lambda-extension/model" + + "github.com/pkg/errors" +) + +type MetadataContainer struct { + Metadata *model.Metadata `json:"metadata"` +} + +func ProcessMetadata(data AgentData, container *MetadataContainer) error { + uncompressedData, err := GetUncompressedBytes(data.Data, data.ContentEncoding) + if err != nil { + return errors.New(fmt.Sprintf("Error uncompressing agent data for metadata extraction : %v", err)) + } + decoder := json.NewDecoder(bytes.NewReader(uncompressedData)) + for { + err = decoder.Decode(container) + if container.Metadata != nil { + Log.Debug("Metadata decoded") + break + } + if err != nil { + if err == io.EOF { + return errors.New("No metadata in current agent transaction") + } else { + return errors.New(fmt.Sprintf("Error uncompressing agent data for metadata extraction : %v", err)) + } + } + } + return nil +} + +func GetUncompressedBytes(rawBytes []byte, encodingType string) ([]byte, error) { + switch encodingType { + case "deflate": + reader := bytes.NewReader([]byte(rawBytes)) + zlibreader, err := zlib.NewReader(reader) + if err != nil { + return nil, fmt.Errorf("could not create zlib.NewReader: %v", err) + } + bodyBytes, err := ioutil.ReadAll(zlibreader) + if err != nil { + return nil, fmt.Errorf("could not read from zlib reader using ioutil.ReadAll: %v", err) + } + return bodyBytes, nil + case "gzip": + reader := bytes.NewReader([]byte(rawBytes)) + zlibreader, err := gzip.NewReader(reader) + if err != nil { + return nil, fmt.Errorf("could not create gzip.NewReader: %v", err) + } + bodyBytes, err := ioutil.ReadAll(zlibreader) + if err != nil { + return nil, fmt.Errorf("could not read from gzip reader using ioutil.ReadAll: %v", err) + } + return bodyBytes, nil + default: + return rawBytes, nil + } +} diff --git a/apm-lambda-extension/extension/process_metadata_test.go b/apm-lambda-extension/extension/process_metadata_test.go new file mode 100644 index 00000000..75188c0c --- /dev/null +++ b/apm-lambda-extension/extension/process_metadata_test.go @@ -0,0 +1,70 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package extension + +import ( + "encoding/json" + "testing" + + "gotest.tools/assert" +) + +func Test_processMetadata(t *testing.T) { + + // Copied from https://github.com/elastic/apm-server/blob/master/testdata/intake-v2/transactions.ndjson. + benchBody := []byte(`{"metadata": {"service": {"name": "1234_service-12a3","node": {"configured_name": "node-123"},"version": "5.1.3","environment": "staging","language": {"name": "ecmascript","version": "8"},"runtime": {"name": "node","version": "8.0.0"},"framework": {"name": "Express","version": "1.2.3"},"agent": {"name": "elastic-node","version": "3.14.0"}},"user": {"id": "123user", "username": "bar", "email": "bar@user.com"}, "labels": {"tag0": null, "tag1": "one", "tag2": 2}, "process": {"pid": 1234,"ppid": 6789,"title": "node","argv": ["node","server.js"]},"system": {"hostname": "prod1.example.com","architecture": "x64","platform": "darwin", "container": {"id": "container-id"}, "kubernetes": {"namespace": "namespace1", "pod": {"uid": "pod-uid", "name": "pod-name"}, "node": {"name": "node-name"}}},"cloud":{"account":{"id":"account_id","name":"account_name"},"availability_zone":"cloud_availability_zone","instance":{"id":"instance_id","name":"instance_name"},"machine":{"type":"machine_type"},"project":{"id":"project_id","name":"project_name"},"provider":"cloud_provider","region":"cloud_region","service":{"name":"lambda"}}}} +{"transaction": { "id": "945254c567a5417e", "trace_id": "0123456789abcdef0123456789abcdef", "parent_id": "abcdefabcdef01234567", "type": "request", "duration": 32.592981, "span_count": { "started": 43 }}} +{"transaction": {"id": "4340a8e0df1906ecbfa9", "trace_id": "0acd456789abcdef0123456789abcdef", "name": "GET /api/types","type": "request","duration": 32.592981,"outcome":"success", "result": "success", "timestamp": 1496170407154000, "sampled": true, "span_count": {"started": 17},"context": {"service": {"runtime": {"version": "7.0"}},"page":{"referer":"http://localhost:8000/test/e2e/","url":"http://localhost:8000/test/e2e/general-usecase/"}, "request": {"socket": {"remote_address": "12.53.12.1","encrypted": true},"http_version": "1.1","method": "POST","url": {"protocol": "https:","full": "https://www.example.com/p/a/t/h?query=string#hash","hostname": "www.example.com","port": "8080","pathname": "/p/a/t/h","search": "?query=string","hash": "#hash","raw": "/p/a/t/h?query=string#hash"},"headers": {"user-agent":["Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36","Mozilla Chrome Edge"],"content-type": "text/html","cookie": "c1=v1, c2=v2","some-other-header": "foo","array": ["foo","bar","baz"]},"cookies": {"c1": "v1","c2": "v2"},"env": {"SERVER_SOFTWARE": "nginx","GATEWAY_INTERFACE": "CGI/1.1"},"body": {"str": "hello world","additional": { "foo": {},"bar": 123,"req": "additional information"}}},"response": {"status_code": 200,"headers": {"content-type": "application/json"},"headers_sent": true,"finished": true,"transfer_size":25.8,"encoded_body_size":26.90,"decoded_body_size":29.90}, "user": {"domain": "ldap://abc","id": "99","username": "foo"},"tags": {"organization_uuid": "9f0e9d64-c185-4d21-a6f4-4673ed561ec8", "tag2": 12, "tag3": 12.45, "tag4": false, "tag5": null },"custom": {"my_key": 1,"some_other_value": "foo bar","and_objects": {"foo": ["bar","baz"]},"(": "not a valid regex and that is fine"}}}} +{"transaction": { "id": "cdef4340a8e0df19", "trace_id": "0acd456789abcdef0123456789abcdef", "type": "request", "duration": 13.980558, "timestamp": 1532976822281000, "sampled": null, "span_count": { "dropped": 55, "started": 436 }, "marks": {"navigationTiming": {"appBeforeBootstrap": 608.9300000000001,"navigationStart": -21},"another_mark": {"some_long": 10,"some_float": 10.0}, "performance": {}}, "context": { "request": { "socket": { "remote_address": "192.0.1", "encrypted": null }, "method": "POST", "headers": { "user-agent": null, "content-type": null, "cookie": null }, "url": { "protocol": null, "full": null, "hostname": null, "port": null, "pathname": null, "search": null, "hash": null, "raw": null } }, "response": { "headers": { "content-type": null } }, "service": {"environment":"testing","name": "service1","node": {"configured_name": "node-ABC"}, "language": {"version": "2.5", "name": "ruby"}, "agent": {"version": "2.2", "name": "elastic-ruby", "ephemeral_id": "justanid"}, "framework": {"version": "5.0", "name": "Rails"}, "version": "2", "runtime": {"version": "2.5", "name": "cruby"}}},"experience":{"cls":1,"fid":2.0,"tbt":3.4,"longtask":{"count":3,"sum":2.5,"max":1}}}} +{"transaction": { "id": "00xxxxFFaaaa1234", "trace_id": "0123456789abcdef0123456789abcdef", "name": "amqp receive", "parent_id": "abcdefabcdef01234567", "type": "messaging", "duration": 3, "span_count": { "started": 1 }, "context": {"message": {"queue": { "name": "new_users"}, "age":{ "ms": 1577958057123}, "headers": {"user_id": "1ax3", "involved_services": ["user", "auth"]}, "body": "user created", "routing_key": "user-created-transaction"}},"session":{"id":"sunday","sequence":123}}} +{"transaction": { "name": "july-2021-delete-after-july-31", "type": "lambda", "result": "success", "id": "142e61450efb8574", "trace_id": "eb56529a1f461c5e7e2f66ecb075e983", "subtype": null, "action": null, "duration": 38.853, "timestamp": 1631736666365048, "sampled": true, "context": { "cloud": { "origin": { "account": { "id": "abc123" }, "provider": "aws", "region": "us-east-1", "service": { "name": "serviceName" } } }, "service": { "origin": { "id": "abc123", "name": "service-name", "version": "1.0" } }, "user": {}, "tags": {}, "custom": { } }, "sync": true, "span_count": { "started": 0 }, "outcome": "unknown", "faas": { "coldstart": false, "execution": "2e13b309-23e1-417f-8bf7-074fc96bc683", "trigger": { "request_id": "FuH2Cir_vHcEMUA=", "type": "http" } }, "sample_rate": 1 } } +`) + + badBenchBody := []byte(`{"metadata": {"service": {"name": "BAD","node": {"configured_name": "node-123"},"version": "5.1.3","environment": "staging","language": {"name": "ecmascript","version": "8"},"runtime": {"name": "node","version": "8.0.0"},"framework": {"name": "Express","version": "1.2.3"},"agent": {"name": "elastic-node","version": "3.14.0"}},"user": {"id": "123user", "username": "bar", "email": "bar@user.com"}, "labels": {"tag0": null, "tag1": "one", "tag2": 2}, "process": {"pid": 1234,"ppid": 6789,"title": "node","argv": ["node","server.js"]},"system": {"hostname": "prod1.example.com","architecture": "x64","platform": "darwin", "container": {"id": "container-id"}, "kubernetes": {"namespace": "namespace1", "pod": {"uid": "pod-uid", "name": "pod-name"}, "node": {"name": "node-name"}}},"cloud":{"account":{"id":"account_id","name":"account_name"},"availability_zone":"cloud_availability_zone","instance":{"id":"instance_id","name":"instance_name"},"machine":{"type":"machine_type"},"project":{"id":"project_id","name":"project_name"},"provider":"cloud_provider","region":"cloud_region","service":{"name":"lambda"}}}} +{"transaction": { "id": "945254c567a5417e", "trace_id": "0123456789abcdef0123456789abcdef", "parent_id": "abcdefabcdef01234567", "type": "request", "duration": 32.592981, "span_count": { "started": 43 }}} +{"transaction": {"id": "4340a8e0df1906ecbfa9", "trace_id": "0acd456789abcdef0123456789abcdef", "name": "GET /api/types","type": "request","duration": 32.592981,"outcome":"success", "result": "success", "timestamp": 1496170407154000, "sampled": true, "span_count": {"started": 17},"context": {"service": {"runtime": {"version": "7.0"}},"page":{"referer":"http://localhost:8000/test/e2e/","url":"http://localhost:8000/test/e2e/general-usecase/"}, "request": {"socket": {"remote_address": "12.53.12.1","encrypted": true},"http_version": "1.1","method": "POST","url": {"protocol": "https:","full": "https://www.example.com/p/a/t/h?query=string#hash","hostname": "www.example.com","port": "8080","pathname": "/p/a/t/h","search": "?query=string","hash": "#hash","raw": "/p/a/t/h?query=string#hash"},"headers": {"user-agent":["Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36","Mozilla Chrome Edge"],"content-type": "text/html","cookie": "c1=v1, c2=v2","some-other-header": "foo","array": ["foo","bar","baz"]},"cookies": {"c1": "v1","c2": "v2"},"env": {"SERVER_SOFTWARE": "nginx","GATEWAY_INTERFACE": "CGI/1.1"},"body": {"str": "hello world","additional": { "foo": {},"bar": 123,"req": "additional information"}}},"response": {"status_code": 200,"headers": {"content-type": "application/json"},"headers_sent": true,"finished": true,"transfer_size":25.8,"encoded_body_size":26.90,"decoded_body_size":29.90}, "user": {"domain": "ldap://abc","id": "99","username": "foo"},"tags": {"organization_uuid": "9f0e9d64-c185-4d21-a6f4-4673ed561ec8", "tag2": 12, "tag3": 12.45, "tag4": false, "tag5": null },"custom": {"my_key": 1,"some_other_value": "foo bar","and_objects": {"foo": ["bar","baz"]},"(": "not a valid regex and that is fine"}}}} +{"transaction": { "id": "cdef4340a8e0df19", "trace_id": "0acd456789abcdef0123456789abcdef", "type": "request", "duration": 13.980558, "timestamp": 1532976822281000, "sampled": null, "span_count": { "dropped": 55, "started": 436 }, "marks": {"navigationTiming": {"appBeforeBootstrap": 608.9300000000001,"navigationStart": -21},"another_mark": {"some_long": 10,"some_float": 10.0}, "performance": {}}, "context": { "request": { "socket": { "remote_address": "192.0.1", "encrypted": null }, "method": "POST", "headers": { "user-agent": null, "content-type": null, "cookie": null }, "url": { "protocol": null, "full": null, "hostname": null, "port": null, "pathname": null, "search": null, "hash": null, "raw": null } }, "response": { "headers": { "content-type": null } }, "service": {"environment":"testing","name": "service1","node": {"configured_name": "node-ABC"}, "language": {"version": "2.5", "name": "ruby"}, "agent": {"version": "2.2", "name": "elastic-ruby", "ephemeral_id": "justanid"}, "framework": {"version": "5.0", "name": "Rails"}, "version": "2", "runtime": {"version": "2.5", "name": "cruby"}}},"experience":{"cls":1,"fid":2.0,"tbt":3.4,"longtask":{"count":3,"sum":2.5,"max":1}}}} +{"transaction": { "id": "00xxxxFFaaaa1234", "trace_id": "0123456789abcdef0123456789abcdef", "name": "amqp receive", "parent_id": "abcdefabcdef01234567", "type": "messaging", "duration": 3, "span_count": { "started": 1 }, "context": {"message": {"queue": { "name": "new_users"}, "age":{ "ms": 1577958057123}, "headers": {"user_id": "1ax3", "involved_services": ["user", "auth"]}, "body": "user created", "routing_key": "user-created-transaction"}},"session":{"id":"sunday","sequence":123}}} +{"transaction": { "name": "july-2021-delete-after-july-31", "type": "lambda", "result": "success", "id": "142e61450efb8574", "trace_id": "eb56529a1f461c5e7e2f66ecb075e983", "subtype": null, "action": null, "duration": 38.853, "timestamp": 1631736666365048, "sampled": true, "context": { "cloud": { "origin": { "account": { "id": "abc123" }, "provider": "aws", "region": "us-east-1", "service": { "name": "serviceName" } } }, "service": { "origin": { "id": "abc123", "name": "service-name", "version": "1.0" } }, "user": {}, "tags": {}, "custom": { } }, "sync": true, "span_count": { "started": 0 }, "outcome": "unknown", "faas": { "coldstart": false, "execution": "2e13b309-23e1-417f-8bf7-074fc96bc683", "trigger": { "request_id": "FuH2Cir_vHcEMUA=", "type": "http" } }, "sample_rate": 1 } } +`) + + agentData := AgentData{Data: benchBody, ContentEncoding: ""} + badAgentData := AgentData{Data: badBenchBody, ContentEncoding: ""} + var mc MetadataContainer + + if mc.Metadata == nil { + if err := ProcessMetadata(agentData, &mc); err != nil { + t.Fail() + } + } + + if mc.Metadata == nil { + if err := ProcessMetadata(badAgentData, &mc); err != nil { + t.Fail() + } + } + + // Metadata is extracted as is. The agent name and version are replaced by values related to the extension in + // logsapi.ProcessMetrics + desiredMetadata := []byte(`{"metadata":{"service":{"name":"1234_service-12a3","version":"5.1.3","environment":"staging","agent":{"name":"elastic-node","version":"3.14.0"},"framework":{"name":"Express","version":"1.2.3"},"language":{"name":"ecmascript","version":"8"},"runtime":{"name":"node","version":"8.0.0"},"node":{"configured_name":"node-123"}},"user":{"username":"bar","id":"123user","email":"bar@user.com"},"labels":{"tag0":null,"tag1":"one","tag2":2},"process":{"pid":1234,"ppid":6789,"title":"node","argv":["node","server.js"]},"system":{"architecture":"x64","hostname":"prod1.example.com","platform":"darwin","container":{"id":"container-id"},"kubernetes":{"namespace":"namespace1","node":{"name":"node-name"},"pod":{"name":"pod-name","uid":"pod-uid"}}},"cloud":{"provider":"cloud_provider","region":"cloud_region","availability_zone":"cloud_availability_zone","instance":{"id":"instance_id","name":"instance_name"},"machine":{"type":"machine_type"},"account":{"id":"account_id","name":"account_name"},"project":{"id":"project_id","name":"project_name"},"service":{"name":"lambda"}}}}`) + extractedMetadata, err := json.Marshal(mc) + if err != nil { + Log.Errorf("Could not marshal extracted metadata : %v", err) + } + assert.DeepEqual(t, desiredMetadata, extractedMetadata) +} From a515a6d456b3600bd1cc8c288c183b82ae2f88e3 Mon Sep 17 00:00:00 2001 From: Jean-Louis Voiseux <48380853+jlvoiseux@users.noreply.github.com> Date: Tue, 31 May 2022 18:58:16 +0200 Subject: [PATCH 04/21] Add Platform metrics processing These changes allow for populating dedicated structs by using the Lambda platform report provided after each invocation. Some fields also need to be added to the Event structs. A notable addition is the modification of the metadata to incorporate extension-specific fields. I think that this information should be included, but the current implementation causes the extension to be displayed as a separate service in the APM UI. --- apm-lambda-extension/extension/client.go | 2 + apm-lambda-extension/logsapi/client.go | 2 + .../logsapi/process_metrics.go | 111 ++++++++++++++++++ .../logsapi/process_metrics_test.go | 92 +++++++++++++++ 4 files changed, 207 insertions(+) create mode 100644 apm-lambda-extension/logsapi/process_metrics.go create mode 100644 apm-lambda-extension/logsapi/process_metrics_test.go diff --git a/apm-lambda-extension/extension/client.go b/apm-lambda-extension/extension/client.go index fd8d41d5..0c6c78b1 100644 --- a/apm-lambda-extension/extension/client.go +++ b/apm-lambda-extension/extension/client.go @@ -23,6 +23,7 @@ import ( "encoding/json" "fmt" "net/http" + "time" ) // RegisterResponse is the body of the response for /register @@ -34,6 +35,7 @@ type RegisterResponse struct { // NextEventResponse is the response for /event/next type NextEventResponse struct { + Timestamp time.Time `json:"timestamp,omitempty"` EventType EventType `json:"eventType"` DeadlineMs int64 `json:"deadlineMs"` RequestID string `json:"requestId"` diff --git a/apm-lambda-extension/logsapi/client.go b/apm-lambda-extension/logsapi/client.go index f8dd490c..093872c0 100644 --- a/apm-lambda-extension/logsapi/client.go +++ b/apm-lambda-extension/logsapi/client.go @@ -62,6 +62,8 @@ const ( // RuntimeDone event is sent when lambda function is finished it's execution RuntimeDone SubEventType = "platform.runtimeDone" Fault SubEventType = "platform.fault" + Report SubEventType = "platform.report" + Start SubEventType = "platform.start" ) // BufferingCfg is the configuration set for receiving logs from Logs API. Whichever of the conditions below is met first, the logs will be sent diff --git a/apm-lambda-extension/logsapi/process_metrics.go b/apm-lambda-extension/logsapi/process_metrics.go new file mode 100644 index 00000000..32e8184d --- /dev/null +++ b/apm-lambda-extension/logsapi/process_metrics.go @@ -0,0 +1,111 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package logsapi + +import ( + "context" + "encoding/json" + "math" + + "elastic/apm-lambda-extension/extension" + "elastic/apm-lambda-extension/model" +) + +type PlatformMetrics struct { + DurationMs float32 `json:"durationMs"` + BilledDurationMs int32 `json:"billedDurationMs"` + MemorySizeMB int32 `json:"memorySizeMB"` + MaxMemoryUsedMB int32 `json:"maxMemoryUsedMB"` + InitDurationMs float32 `json:"initDurationMs"` +} + +type MetricsContainer struct { + Metrics *model.Metrics `json:"metricset"` +} + +// Add adds a metric with the given name, labels, and value, +// The labels are expected to be sorted lexicographically. +func (mc MetricsContainer) Add(name string, labels []model.MetricLabel, value float64) { + mc.addMetric(name, model.Metric{Value: value}) +} + +// Simplified version of https://github.com/elastic/apm-agent-go/blob/675e8398c7fe546f9fd169bef971b9ccfbcdc71f/metrics.go#L89 +func (mc MetricsContainer) addMetric(name string, metric model.Metric) { + + if mc.Metrics.Samples == nil { + mc.Metrics.Samples = make(map[string]model.Metric) + } + mc.Metrics.Samples[name] = metric +} + +func ProcessPlatformReport(ctx context.Context, metadataContainer *extension.MetadataContainer, functionData *extension.NextEventResponse, platformReport LogEvent) (extension.AgentData, error) { + var metricsData []byte + metricsContainer := MetricsContainer{ + Metrics: &model.Metrics{}, + } + convMB2Bytes := float64(1024 * 1024) + platformReportMetrics := platformReport.Record.Metrics + + // APM Spec Fields + // Timestamp + metricsContainer.Metrics.Timestamp = platformReport.Time.UnixMicro() + // FaaS Fields + metricsContainer.Metrics.Labels = make(map[string]interface{}) + metricsContainer.Metrics.Labels["faas.execution"] = platformReport.Record.RequestId + metricsContainer.Metrics.Labels["faas.id"] = functionData.InvokedFunctionArn + if platformReportMetrics.InitDurationMs > 0 { + metricsContainer.Metrics.Labels["faas.coldstart"] = true + } else { + metricsContainer.Metrics.Labels["faas.coldstart"] = false + } + // System + metricsContainer.Add("system.memory.total", nil, float64(platformReportMetrics.MemorySizeMB)*convMB2Bytes) // Unit : Bytes + metricsContainer.Add("system.memory.actual.free", nil, float64(platformReportMetrics.MemorySizeMB-platformReportMetrics.MaxMemoryUsedMB)*convMB2Bytes) // Unit : Bytes + + // Raw Metrics + // AWS uses binary multiples to compute memory : https://aws.amazon.com/about-aws/whats-new/2020/12/aws-lambda-supports-10gb-memory-6-vcpu-cores-lambda-functions/ + metricsContainer.Add("aws.lambda.metrics.TotalMemory", nil, float64(platformReportMetrics.MemorySizeMB)*convMB2Bytes) // Unit : Bytes + metricsContainer.Add("aws.lambda.metrics.UsedMemory", nil, float64(platformReportMetrics.MaxMemoryUsedMB)*convMB2Bytes) // Unit : Bytes + metricsContainer.Add("aws.lambda.metrics.Duration", nil, float64(platformReportMetrics.DurationMs)) // Unit : Milliseconds + metricsContainer.Add("aws.lambda.metrics.BilledDuration", nil, float64(platformReportMetrics.BilledDurationMs)) // Unit : Milliseconds + metricsContainer.Add("aws.lambda.metrics.ColdStartDuration", nil, float64(platformReportMetrics.InitDurationMs)) // Unit : Milliseconds + // In AWS Lambda, the Timeout is configured as an integer number of seconds. We use this assumption to derive the Timeout from + // - The epoch corresponding to the end of the current invocation (its "deadline") + // - The epoch corresponding to the start of the current invocation + // - The multiplication / division then rounds the value to obtain a number of ms that can be expressed a multiple of 1000 (see initial assumption) + metricsContainer.Add("aws.lambda.metrics.Timeout", nil, math.Ceil(float64(functionData.DeadlineMs-functionData.Timestamp.UnixMilli())/1e3)*1e3) // Unit : Milliseconds + + metricsJson, err := json.Marshal(metricsContainer) + if err != nil { + return extension.AgentData{}, err + } + + if metadataContainer.Metadata != nil { + //TODO : Discuss relevance of displaying extension name + metadataContainer.Metadata.Service.Agent.Name = "apm-lambda-extension" + metadataContainer.Metadata.Service.Agent.Version = extension.Version + metadataJson, err := json.Marshal(metadataContainer) + if err != nil { + return extension.AgentData{}, err + } + metricsData = append(metadataJson, []byte("\n")...) + } + + metricsData = append(metricsData, metricsJson...) + return extension.AgentData{Data: metricsData}, nil +} diff --git a/apm-lambda-extension/logsapi/process_metrics_test.go b/apm-lambda-extension/logsapi/process_metrics_test.go new file mode 100644 index 00000000..343e101f --- /dev/null +++ b/apm-lambda-extension/logsapi/process_metrics_test.go @@ -0,0 +1,92 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package logsapi + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "elastic/apm-lambda-extension/extension" + "elastic/apm-lambda-extension/model" + + "gotest.tools/assert" +) + +func Test_processPlatformReport(t *testing.T) { + + timestamp := time.Now() + + pm := PlatformMetrics{ + DurationMs: 182.43, + BilledDurationMs: 183, + MemorySizeMB: 128, + MaxMemoryUsedMB: 76, + InitDurationMs: 422.97, + } + + logEventRecord := LogEventRecord{ + RequestId: "6f7f0961f83442118a7af6fe80b88d56", + Status: "Available", + Metrics: pm, + } + + logEvent := LogEvent{ + Time: timestamp, + Type: "platform.report", + StringRecord: "", + Record: logEventRecord, + } + + event := extension.NextEventResponse{ + Timestamp: timestamp, + EventType: extension.Invoke, + DeadlineMs: timestamp.UnixNano()/1e6 + 4584, // Milliseconds + RequestID: "8476a536-e9f4-11e8-9739-2dfe598c3fcd", + InvokedFunctionArn: "arn:aws:lambda:us-east-2:123456789012:function:custom-runtime", + Tracing: extension.Tracing{ + Type: "None", + Value: "None", + }, + } + + desiredOutput := fmt.Sprintf(`{"metadata":{"service":{"agent":{"name":"apm-lambda-extension","version":"%s"},"framework":{"name":"AWS Lambda","version":""},"language":{"name":"python","version":"3.9.8"},"runtime":{"name":"","version":""},"node":{}},"user":{},"process":{"pid":0},"system":{"container":{"id":""},"kubernetes":{"node":{},"pod":{}}},"cloud":{"provider":"","instance":{},"machine":{},"account":{},"project":{},"service":{}}}} +{"metricset":{"timestamp":%d,"transaction":{},"span":{},"tags":{"faas.coldstart":true,"faas.execution":"6f7f0961f83442118a7af6fe80b88d56","faas.id":"arn:aws:lambda:us-east-2:123456789012:function:custom-runtime"},"samples":{"aws.lambda.metrics.BilledDuration":{"value":183},"aws.lambda.metrics.ColdStartDuration":{"value":422.9700012207031},"aws.lambda.metrics.Duration":{"value":182.42999267578125},"aws.lambda.metrics.Timeout":{"value":5000},"aws.lambda.metrics.TotalMemory":{"value":134217728},"aws.lambda.metrics.UsedMemory":{"value":79691776},"system.memory.actual.free":{"value":54525952},"system.memory.total":{"value":134217728}}}}`, extension.Version, timestamp.UnixNano()/1e3) + + mc := extension.MetadataContainer{ + Metadata: &model.Metadata{}, + } + mc.Metadata.Service = model.Service{ + Name: os.Getenv("AWS_LAMBDA_FUNCTION_NAME"), + Agent: model.Agent{Name: "python", Version: "6.7.2"}, + Language: model.Language{Name: "python", Version: "3.9.8"}, + Runtime: model.Runtime{Name: os.Getenv("AWS_EXECUTION_ENV")}, + Framework: model.Framework{Name: "AWS Lambda"}, + } + mc.Metadata.Process = model.Process{} + mc.Metadata.System = model.System{} + + rawBytes, err := ProcessPlatformReport(context.Background(), &mc, &event, logEvent) + if err != nil { + t.Fail() + } + requestBytes, err := extension.GetUncompressedBytes(rawBytes.Data, "") + assert.Equal(t, string(requestBytes), desiredOutput) +} From c2b056a07d6666ac300fa8303d1945c9cbae5700 Mon Sep 17 00:00:00 2001 From: Jean-Louis Voiseux <48380853+jlvoiseux@users.noreply.github.com> Date: Tue, 31 May 2022 19:02:20 +0200 Subject: [PATCH 05/21] Update logs API to handle Lambda platform metrics --- .../logsapi/route_handlers.go | 3 +- .../logsapi/route_handlers_test.go | 7 ++++ apm-lambda-extension/logsapi/subscribe.go | 39 +++++++++++++++---- 3 files changed, 41 insertions(+), 8 deletions(-) diff --git a/apm-lambda-extension/logsapi/route_handlers.go b/apm-lambda-extension/logsapi/route_handlers.go index 3da4c316..7606f654 100644 --- a/apm-lambda-extension/logsapi/route_handlers.go +++ b/apm-lambda-extension/logsapi/route_handlers.go @@ -18,10 +18,11 @@ package logsapi import ( - "elastic/apm-lambda-extension/extension" "encoding/json" "net/http" "time" + + "elastic/apm-lambda-extension/extension" ) func handleLogEventsRequest(transport *LogsTransport) func(w http.ResponseWriter, r *http.Request) { diff --git a/apm-lambda-extension/logsapi/route_handlers_test.go b/apm-lambda-extension/logsapi/route_handlers_test.go index 9bb2f6e3..f6fde29d 100644 --- a/apm-lambda-extension/logsapi/route_handlers_test.go +++ b/apm-lambda-extension/logsapi/route_handlers_test.go @@ -48,6 +48,13 @@ func TestLogEventUnmarshalReport(t *testing.T) { rec := LogEventRecord{ RequestId: "6f7f0961f83442118a7af6fe80b88d56", Status: "", // no status was given in sample json + Metrics: PlatformMetrics{ + DurationMs: 101.51, + BilledDurationMs: 300, + MemorySizeMB: 512, + MaxMemoryUsedMB: 33, + InitDurationMs: 116.67, + }, } assert.Equal(t, rec, le.Record) diff --git a/apm-lambda-extension/logsapi/subscribe.go b/apm-lambda-extension/logsapi/subscribe.go index 3d844bd3..d93ba0f5 100644 --- a/apm-lambda-extension/logsapi/subscribe.go +++ b/apm-lambda-extension/logsapi/subscribe.go @@ -19,13 +19,14 @@ package logsapi import ( "context" - "elastic/apm-lambda-extension/extension" "fmt" "net" "net/http" "os" "time" + "elastic/apm-lambda-extension/extension" + "github.com/pkg/errors" ) @@ -57,8 +58,9 @@ type LogEvent struct { // LogEventRecord is a sub-object in a Logs API event type LogEventRecord struct { - RequestId string `json:"requestId"` - Status string `json:"status"` + RequestId string `json:"requestId"` + Status string `json:"status"` + Metrics PlatformMetrics `json:"metrics"` } // Subscribes to the Logs API @@ -156,16 +158,25 @@ func checkLambdaFunction() bool { return false } -// WaitRuntimeDone consumes events until a RuntimeDone event corresponding +// ProcessLogs consumes events until a RuntimeDone event corresponding // to requestID is received, or ctx is cancelled, and then returns. -func WaitRuntimeDone(ctx context.Context, requestID string, transport *LogsTransport, runtimeDoneSignal chan struct{}) error { +func ProcessLogs( + ctx context.Context, + requestID string, + apmServerTransport *extension.ApmServerTransport, + logsTransport *LogsTransport, + metadataContainer *extension.MetadataContainer, + runtimeDoneSignal chan struct{}, + prevEvent *extension.NextEventResponse, +) error { for { select { - case logEvent := <-transport.logsChannel: + case logEvent := <-logsTransport.logsChannel: extension.Log.Debugf("Received log event %v", logEvent.Type) + switch logEvent.Type { // Check the logEvent for runtimeDone and compare the RequestID // to the id that came in via the Next API - if logEvent.Type == RuntimeDone { + case RuntimeDone: if logEvent.Record.RequestId == requestID { extension.Log.Info("Received runtimeDone event for this function invocation") runtimeDoneSignal <- struct{}{} @@ -173,6 +184,20 @@ func WaitRuntimeDone(ctx context.Context, requestID string, transport *LogsTrans } else { extension.Log.Debug("Log API runtimeDone event request id didn't match") } + // Check if the logEvent contains metrics and verify that they can be linked to the previous invocation + case Report: + if prevEvent != nil && logEvent.Record.RequestId == prevEvent.RequestID { + extension.Log.Debug("Received platform report for the previous function invocation") + processedMetrics, err := ProcessPlatformReport(ctx, metadataContainer, prevEvent, logEvent) + if err != nil { + extension.Log.Errorf("Error processing Lambda platform metrics : %v", err) + } else { + apmServerTransport.EnqueueAPMData(processedMetrics) + } + } else { + extension.Log.Warn("report event request id didn't match the previous event id") + extension.Log.Debug("Log API runtimeDone event request id didn't match") + } } case <-ctx.Done(): extension.Log.Debug("Current invocation over. Interrupting logs processing goroutine") From 3ec3bf9f4c9968c42113593ff91a484f0a4b9e29 Mon Sep 17 00:00:00 2001 From: Jean-Louis Voiseux <48380853+jlvoiseux@users.noreply.github.com> Date: Tue, 31 May 2022 19:03:20 +0200 Subject: [PATCH 06/21] Update APM Transport to handle metadata population --- apm-lambda-extension/extension/apm_server_transport.go | 8 +++++++- .../extension/apm_server_transport_test.go | 3 ++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/apm-lambda-extension/extension/apm_server_transport.go b/apm-lambda-extension/extension/apm_server_transport.go index 9b519098..53a29984 100644 --- a/apm-lambda-extension/extension/apm_server_transport.go +++ b/apm-lambda-extension/extension/apm_server_transport.go @@ -75,7 +75,7 @@ func InitApmServerTransport(config *extensionConfig) *ApmServerTransport { // StartBackgroundApmDataForwarding Receive agent data as it comes in and post it to the APM server. // Stop checking for, and sending agent data when the function invocation // has completed, signaled via a channel. -func (transport *ApmServerTransport) ForwardApmData(ctx context.Context) error { +func (transport *ApmServerTransport) ForwardApmData(ctx context.Context, metadataContainer *MetadataContainer) error { if transport.status == Failing { return nil } @@ -85,6 +85,12 @@ func (transport *ApmServerTransport) ForwardApmData(ctx context.Context) error { Log.Debug("Invocation context cancelled, not processing any more agent data") return nil case agentData := <-transport.dataChannel: + if metadataContainer.Metadata == nil { + err := ProcessMetadata(agentData, metadataContainer) + if err != nil { + Log.Errorf("Error extracting metadata from agent payload %v", err) + } + } if err := transport.PostToApmServer(ctx, agentData); err != nil { return fmt.Errorf("error sending to APM server, skipping: %v", err) } diff --git a/apm-lambda-extension/extension/apm_server_transport_test.go b/apm-lambda-extension/extension/apm_server_transport_test.go index 5362e588..eab30a57 100644 --- a/apm-lambda-extension/extension/apm_server_transport_test.go +++ b/apm-lambda-extension/extension/apm_server_transport_test.go @@ -20,12 +20,13 @@ package extension import ( "compress/gzip" "context" - "github.com/stretchr/testify/assert" "io" "io/ioutil" "net/http" "net/http/httptest" "testing" + + "github.com/stretchr/testify/assert" ) func TestPostToApmServerDataCompressed(t *testing.T) { From 2b651c8d239bc930125bbaf6653336c1406035f2 Mon Sep 17 00:00:00 2001 From: Jean-Louis Voiseux <48380853+jlvoiseux@users.noreply.github.com> Date: Tue, 31 May 2022 19:03:56 +0200 Subject: [PATCH 07/21] Update main to handle metadata and metrics --- apm-lambda-extension/main.go | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/apm-lambda-extension/main.go b/apm-lambda-extension/main.go index 5e7d9f58..bbf82bc1 100644 --- a/apm-lambda-extension/main.go +++ b/apm-lambda-extension/main.go @@ -85,24 +85,40 @@ func main() { extension.Log.Warnf("Error while subscribing to the Logs API: %v", err) } + // The previous event id is used to validate the received Lambda metrics + var prevEvent *extension.NextEventResponse + // This data structure contains metadata tied to the current Lambda instance. If empty, it is populated once for each + // active Lambda environment + metadataContainer := extension.MetadataContainer{} + for { select { case <-ctx.Done(): return default: var backgroundDataSendWg sync.WaitGroup - processEvent(ctx, cancel, apmServerTransport, logsTransport, &backgroundDataSendWg) + event := processEvent(ctx, cancel, apmServerTransport, logsTransport, &backgroundDataSendWg, prevEvent, &metadataContainer) extension.Log.Debug("Waiting for background data send to end") backgroundDataSendWg.Wait() if config.SendStrategy == extension.SyncFlush { // Flush APM data now that the function invocation has completed apmServerTransport.FlushAPMData(ctx) } + prevEvent = event } } } -func processEvent(ctx context.Context, cancel context.CancelFunc, apmServerTransport *extension.ApmServerTransport, logsTransport *logsapi.LogsTransport, backgroundDataSendWg *sync.WaitGroup) { +func processEvent( + ctx context.Context, + cancel context.CancelFunc, + apmServerTransport *extension.ApmServerTransport, + logsTransport *logsapi.LogsTransport, + backgroundDataSendWg *sync.WaitGroup, + prevEvent *extension.NextEventResponse, + metadataContainer *extension.MetadataContainer, +) *extension.NextEventResponse { + // Invocation context invocationCtx, invocationCancel := context.WithCancel(ctx) defer invocationCancel() @@ -119,7 +135,7 @@ func processEvent(ctx context.Context, cancel context.CancelFunc, apmServerTrans extension.Log.Errorf("Error: %s", err) extension.Log.Infof("Exit signal sent to runtime : %s", status) extension.Log.Infof("Exiting") - return + return nil } extension.Log.Debug("Received event.") @@ -127,7 +143,7 @@ func processEvent(ctx context.Context, cancel context.CancelFunc, apmServerTrans if event.EventType == extension.Shutdown { cancel() - return + return event } // APM Data Processing @@ -136,17 +152,17 @@ func processEvent(ctx context.Context, cancel context.CancelFunc, apmServerTrans backgroundDataSendWg.Add(1) go func() { defer backgroundDataSendWg.Done() - if err := apmServerTransport.ForwardApmData(invocationCtx); err != nil { + if err := apmServerTransport.ForwardApmData(invocationCtx, metadataContainer); err != nil { extension.Log.Error(err) } }() - // Lambda Service Logs Processing + // Lambda Service Logs Processing, also used to extract metrics from APM logs // This goroutine should not be started if subscription failed runtimeDone := make(chan struct{}) if logsTransport != nil { go func() { - if err := logsapi.WaitRuntimeDone(invocationCtx, event.RequestID, logsTransport, runtimeDone); err != nil { + if err := logsapi.ProcessLogs(invocationCtx, event.RequestID, apmServerTransport, logsTransport, metadataContainer, runtimeDone, prevEvent); err != nil { extension.Log.Errorf("Error while processing Lambda Logs ; %v", err) } else { close(runtimeDone) @@ -179,4 +195,6 @@ func processEvent(ctx context.Context, cancel context.CancelFunc, apmServerTrans case <-timer.C: extension.Log.Info("Time expired waiting for agent signal or runtimeDone event") } + + return event } From 614be84d8f0930c36d73a6d813fb8f745eab7ace Mon Sep 17 00:00:00 2001 From: Jean-Louis Voiseux <48380853+jlvoiseux@users.noreply.github.com> Date: Tue, 31 May 2022 19:14:29 +0200 Subject: [PATCH 08/21] Add end-to-end testing for metrics and metadata --- apm-lambda-extension/extension/logger_test.go | 7 +- apm-lambda-extension/main_test.go | 75 ++++++++++++++++--- 2 files changed, 69 insertions(+), 13 deletions(-) diff --git a/apm-lambda-extension/extension/logger_test.go b/apm-lambda-extension/extension/logger_test.go index d869c808..05d5cedd 100644 --- a/apm-lambda-extension/extension/logger_test.go +++ b/apm-lambda-extension/extension/logger_test.go @@ -18,12 +18,13 @@ package extension import ( - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.uber.org/zap/zapcore" "io/ioutil" "os" "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap/zapcore" ) func init() { diff --git a/apm-lambda-extension/main_test.go b/apm-lambda-extension/main_test.go index 8355f9f3..3f231c82 100644 --- a/apm-lambda-extension/main_test.go +++ b/apm-lambda-extension/main_test.go @@ -20,12 +20,8 @@ package main import ( "bytes" "context" - e2eTesting "elastic/apm-lambda-extension/e2e-testing" - "elastic/apm-lambda-extension/extension" - "elastic/apm-lambda-extension/logsapi" "encoding/json" "fmt" - "github.com/stretchr/testify/suite" "io/ioutil" "net" "net/http" @@ -36,6 +32,12 @@ import ( "testing" "time" + e2eTesting "elastic/apm-lambda-extension/e2e-testing" + "elastic/apm-lambda-extension/extension" + "elastic/apm-lambda-extension/logsapi" + + "github.com/stretchr/testify/suite" + "github.com/google/uuid" "github.com/stretchr/testify/assert" ) @@ -46,7 +48,8 @@ const ( InvokeHang MockEventType = "Hang" InvokeStandard MockEventType = "Standard" InvokeStandardInfo MockEventType = "StandardInfo" - InvokeStandardFlush MockEventType = "Flush" + InvokeStandardFlush MockEventType = "StandardFlush" + InvokeStandardMetadata MockEventType = "StandardMetadata" InvokeLateFlush MockEventType = "LateFlush" InvokeWaitgroupsRace MockEventType = "InvokeWaitgroupsRace" InvokeMultipleTransactionsOverload MockEventType = "MultipleTransactionsOverload" @@ -113,8 +116,6 @@ func initMockServers(eventsChannel chan MockEvent) (*httptest.Server, *httptest. case Crashes: panic("Server crashed") default: - w.WriteHeader(http.StatusInternalServerError) - return } if r.RequestURI == "/intake/v2/events" { w.WriteHeader(http.StatusAccepted) @@ -206,9 +207,10 @@ func initMockServers(eventsChannel chan MockEvent) (*httptest.Server, *httptest. } func processMockEvent(currId string, event MockEvent, extensionPort string, internals *MockServerInternals) { - sendLogEvent(currId, "platform.start") + sendLogEvent(currId, logsapi.Start) client := http.Client{} sendRuntimeDone := true + sendMetrics := true switch event.Type { case InvokeHang: time.Sleep(time.Duration(event.Timeout) * time.Second) @@ -217,6 +219,12 @@ func processMockEvent(currId string, event MockEvent, extensionPort string, inte req, _ := http.NewRequest("POST", fmt.Sprintf("http://localhost:%s/intake/v2/events", extensionPort), bytes.NewBuffer([]byte(event.APMServerBehavior))) res, _ := client.Do(req) extension.Log.Debugf("Response seen by the agent : %d", res.StatusCode) + case InvokeStandardMetadata: + time.Sleep(time.Duration(event.ExecutionDuration) * time.Second) + metadata := `{"metadata":{"service":{"name":"1234_service-12a3","version":"5.1.3","environment":"staging","agent":{"name":"elastic-node","version":"3.14.0"},"framework":{"name":"Express","version":"1.2.3"},"language":{"name":"ecmascript","version":"8"},"runtime":{"name":"node","version":"8.0.0"},"node":{"configured_name":"node-123"}},"user":{"username":"bar","id":"123user","email":"bar@user.com"},"labels":{"tag0":null,"tag1":"one","tag2":2},"process":{"pid":1234,"ppid":6789,"title":"node","argv":["node","server.js"]},"system":{"architecture":"x64","hostname":"prod1.example.com","platform":"darwin","container":{"id":"container-id"},"kubernetes":{"namespace":"namespace1","node":{"name":"node-name"},"pod":{"name":"pod-name","uid":"pod-uid"}}},"cloud":{"provider":"cloud_provider","region":"cloud_region","availability_zone":"cloud_availability_zone","instance":{"id":"instance_id","name":"instance_name"},"machine":{"type":"machine_type"},"account":{"id":"account_id","name":"account_name"},"project":{"id":"project_id","name":"project_name"},"service":{"name":"lambda"}}}}` + req, _ := http.NewRequest("POST", fmt.Sprintf("http://localhost:%s/intake/v2/events", extensionPort), bytes.NewBuffer([]byte(metadata))) + res, _ := client.Do(req) + extension.Log.Debugf("Response seen by the agent : %d", res.StatusCode) case InvokeStandardFlush: time.Sleep(time.Duration(event.ExecutionDuration) * time.Second) reqData, _ := http.NewRequest("POST", fmt.Sprintf("http://localhost:%s/intake/v2/events?flushed=true", extensionPort), bytes.NewBuffer([]byte(event.APMServerBehavior))) @@ -233,6 +241,8 @@ func processMockEvent(currId string, event MockEvent, extensionPort string, inte } internals.WaitGroup.Done() }() + // For this specific scenario, we do not want to see metrics in the APM Server logs (in order to easily check if the logs contain to "TimelyResponse" back to back). + sendMetrics = false case InvokeWaitgroupsRace: time.Sleep(time.Duration(event.ExecutionDuration) * time.Second) reqData0, _ := http.NewRequest("POST", fmt.Sprintf("http://localhost:%s/intake/v2/events", extensionPort), bytes.NewBuffer([]byte(event.APMServerBehavior))) @@ -275,7 +285,10 @@ func processMockEvent(currId string, event MockEvent, extensionPort string, inte case Shutdown: } if sendRuntimeDone { - sendLogEvent(currId, "platform.runtimeDone") + sendLogEvent(currId, logsapi.RuntimeDone) + } + if sendMetrics { + sendLogEvent(currId, logsapi.Report) } } @@ -300,6 +313,16 @@ func sendLogEvent(requestId string, logEventType logsapi.SubEventType) { record := logsapi.LogEventRecord{ RequestId: requestId, } + if logEventType == logsapi.Report { + record.Metrics = logsapi.PlatformMetrics{ + BilledDurationMs: 60, + DurationMs: 59.9, + MemorySizeMB: 128, + MaxMemoryUsedMB: 60, + InitDurationMs: 500, + } + } + logEvent := logsapi.LogEvent{ Time: time.Now(), Type: logEventType, @@ -440,7 +463,6 @@ func (suite *MainUnitTestsSuite) TestAPMServerRecovery() { if err := os.Setenv("ELASTIC_APM_DATA_FORWARDER_TIMEOUT_SECONDS", "1"); err != nil { suite.T().Fail() } - eventsChain := []MockEvent{ {Type: InvokeStandard, APMServerBehavior: Hangs, ExecutionDuration: 1, Timeout: 5}, {Type: InvokeStandard, APMServerBehavior: TimelyResponse, ExecutionDuration: 1, Timeout: 5}, @@ -531,3 +553,36 @@ func (suite *MainUnitTestsSuite) TestInfoRequestHangs() { assert.False(suite.T(), strings.Contains(suite.lambdaServerInternals.Data, "7814d524d3602e70b703539c57568cba6964fc20")) suite.apmServerInternals.UnlockSignalChannel <- struct{}{} } + +// TestMetricsWithoutMetadata checks if the extension sends metrics corresponding to invocation n during invocation +// n+1, even if the metadata container was not populated +func (suite *MainUnitTestsSuite) TestMetricsWithoutMetadata() { + eventsChain := []MockEvent{ + {Type: InvokeStandard, APMServerBehavior: TimelyResponse, ExecutionDuration: 1, Timeout: 5}, + {Type: InvokeStandard, APMServerBehavior: TimelyResponse, ExecutionDuration: 1, Timeout: 5}, + } + eventQueueGenerator(eventsChain, suite.eventsChannel) + assert.NotPanics(suite.T(), main) + assert.Contains(suite.T(), suite.apmServerInternals.Data, `aws.lambda.metrics.BilledDuration":{"value":60`) + assert.Contains(suite.T(), suite.apmServerInternals.Data, `aws.lambda.metrics.Duration":{"value":59.9`) + assert.Contains(suite.T(), suite.apmServerInternals.Data, `aws.lambda.metrics.TotalMemory":{"value":134217728`) + assert.Contains(suite.T(), suite.apmServerInternals.Data, `aws.lambda.metrics.UsedMemory":{"value":62914560`) + assert.Contains(suite.T(), suite.apmServerInternals.Data, `aws.lambda.metrics.ColdStartDuration":{"value":500`) +} + +// TestMetricsWithMetadata checks if the extension sends metrics corresponding to invocation n during invocation +// n+1, even if the metadata container was not populated +func (suite *MainUnitTestsSuite) TestMetricsWithMetadata() { + eventsChain := []MockEvent{ + {Type: InvokeStandardMetadata, APMServerBehavior: TimelyResponse, ExecutionDuration: 1, Timeout: 5}, + {Type: InvokeStandardMetadata, APMServerBehavior: TimelyResponse, ExecutionDuration: 1, Timeout: 5}, + } + eventQueueGenerator(eventsChain, suite.eventsChannel) + assert.NotPanics(suite.T(), main) + assert.Contains(suite.T(), suite.apmServerInternals.Data, fmt.Sprintf(`{"metadata":{"service":{"name":"1234_service-12a3","version":"5.1.3","environment":"staging","agent":{"name":"apm-lambda-extension","version":"%s"},"framework":{"name":"Express","version":"1.2.3"},"language":{"name":"ecmascript","version":"8"},"runtime":{"name":"node","version":"8.0.0"},"node":{"configured_name":"node-123"}},"user":{"username":"bar","id":"123user","email":"bar@user.com"},"labels":{"tag0":null,"tag1":"one","tag2":2},"process":{"pid":1234,"ppid":6789,"title":"node","argv":["node","server.js"]},"system":{"architecture":"x64","hostname":"prod1.example.com","platform":"darwin","container":{"id":"container-id"},"kubernetes":{"namespace":"namespace1","node":{"name":"node-name"},"pod":{"name":"pod-name","uid":"pod-uid"}}},"cloud":{"provider":"cloud_provider","region":"cloud_region","availability_zone":"cloud_availability_zone","instance":{"id":"instance_id","name":"instance_name"},"machine":{"type":"machine_type"},"account":{"id":"account_id","name":"account_name"},"project":{"id":"project_id","name":"project_name"},"service":{"name":"lambda"}}}}`, extension.Version)) + assert.Contains(suite.T(), suite.apmServerInternals.Data, `aws.lambda.metrics.BilledDuration":{"value":60`) + assert.Contains(suite.T(), suite.apmServerInternals.Data, `aws.lambda.metrics.Duration":{"value":59.9`) + assert.Contains(suite.T(), suite.apmServerInternals.Data, `aws.lambda.metrics.TotalMemory":{"value":134217728`) + assert.Contains(suite.T(), suite.apmServerInternals.Data, `aws.lambda.metrics.UsedMemory":{"value":62914560`) + assert.Contains(suite.T(), suite.apmServerInternals.Data, `aws.lambda.metrics.ColdStartDuration":{"value":500`) +} From 3f785e9c1905bd973ee04144e03cbdcce926a95e Mon Sep 17 00:00:00 2001 From: Jean-Louis Voiseux <48380853+jlvoiseux@users.noreply.github.com> Date: Tue, 31 May 2022 20:18:27 +0200 Subject: [PATCH 09/21] Fix: Populate event timestamp --- apm-lambda-extension/main.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apm-lambda-extension/main.go b/apm-lambda-extension/main.go index bbf82bc1..0a452f19 100644 --- a/apm-lambda-extension/main.go +++ b/apm-lambda-extension/main.go @@ -138,6 +138,8 @@ func processEvent( return nil } + // Used to compute Lambda Timeout + event.Timestamp = time.Now() extension.Log.Debug("Received event.") extension.Log.Debugf("%v", extension.PrettyPrint(event)) From d03005e73348f51dc9b64ba65d82f4e5db226be8 Mon Sep 17 00:00:00 2001 From: Jean-Louis Voiseux <48380853+jlvoiseux@users.noreply.github.com> Date: Tue, 31 May 2022 20:33:47 +0200 Subject: [PATCH 10/21] Fix: linting --- apm-lambda-extension/logsapi/process_metrics_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apm-lambda-extension/logsapi/process_metrics_test.go b/apm-lambda-extension/logsapi/process_metrics_test.go index 343e101f..e5abf5ad 100644 --- a/apm-lambda-extension/logsapi/process_metrics_test.go +++ b/apm-lambda-extension/logsapi/process_metrics_test.go @@ -88,5 +88,8 @@ func Test_processPlatformReport(t *testing.T) { t.Fail() } requestBytes, err := extension.GetUncompressedBytes(rawBytes.Data, "") + if err != nil { + t.Fail() + } assert.Equal(t, string(requestBytes), desiredOutput) } From 11e874b95f8db509cc7d0a6b82306ec751b6d564 Mon Sep 17 00:00:00 2001 From: Jean-Louis Voiseux <48380853+jlvoiseux@users.noreply.github.com> Date: Thu, 2 Jun 2022 15:40:41 +0200 Subject: [PATCH 11/21] Fix: Implement review comments (unit tests) --- .../extension/process_metadata_test.go | 17 +++++------------ .../logsapi/process_metrics_test.go | 17 +++++++---------- 2 files changed, 12 insertions(+), 22 deletions(-) diff --git a/apm-lambda-extension/extension/process_metadata_test.go b/apm-lambda-extension/extension/process_metadata_test.go index 75188c0c..4700f5a5 100644 --- a/apm-lambda-extension/extension/process_metadata_test.go +++ b/apm-lambda-extension/extension/process_metadata_test.go @@ -21,6 +21,7 @@ import ( "encoding/json" "testing" + "github.com/stretchr/testify/require" "gotest.tools/assert" ) @@ -35,7 +36,8 @@ func Test_processMetadata(t *testing.T) { {"transaction": { "name": "july-2021-delete-after-july-31", "type": "lambda", "result": "success", "id": "142e61450efb8574", "trace_id": "eb56529a1f461c5e7e2f66ecb075e983", "subtype": null, "action": null, "duration": 38.853, "timestamp": 1631736666365048, "sampled": true, "context": { "cloud": { "origin": { "account": { "id": "abc123" }, "provider": "aws", "region": "us-east-1", "service": { "name": "serviceName" } } }, "service": { "origin": { "id": "abc123", "name": "service-name", "version": "1.0" } }, "user": {}, "tags": {}, "custom": { } }, "sync": true, "span_count": { "started": 0 }, "outcome": "unknown", "faas": { "coldstart": false, "execution": "2e13b309-23e1-417f-8bf7-074fc96bc683", "trigger": { "request_id": "FuH2Cir_vHcEMUA=", "type": "http" } }, "sample_rate": 1 } } `) - badBenchBody := []byte(`{"metadata": {"service": {"name": "BAD","node": {"configured_name": "node-123"},"version": "5.1.3","environment": "staging","language": {"name": "ecmascript","version": "8"},"runtime": {"name": "node","version": "8.0.0"},"framework": {"name": "Express","version": "1.2.3"},"agent": {"name": "elastic-node","version": "3.14.0"}},"user": {"id": "123user", "username": "bar", "email": "bar@user.com"}, "labels": {"tag0": null, "tag1": "one", "tag2": 2}, "process": {"pid": 1234,"ppid": 6789,"title": "node","argv": ["node","server.js"]},"system": {"hostname": "prod1.example.com","architecture": "x64","platform": "darwin", "container": {"id": "container-id"}, "kubernetes": {"namespace": "namespace1", "pod": {"uid": "pod-uid", "name": "pod-name"}, "node": {"name": "node-name"}}},"cloud":{"account":{"id":"account_id","name":"account_name"},"availability_zone":"cloud_availability_zone","instance":{"id":"instance_id","name":"instance_name"},"machine":{"type":"machine_type"},"project":{"id":"project_id","name":"project_name"},"provider":"cloud_provider","region":"cloud_region","service":{"name":"lambda"}}}} + // The number of closing curly braces does not match - this body is not supposed to be unmarshaled successfully, which is accordingly tested. + badBenchBody := []byte(`{"metadata": {"service": {"name": "BAD","node": {"configured_name": "node-123"},"version": "5.1.3","environment": "staging","language": {"name": "ecmascript","version": "8"},"runtime": {"name": "node","version": "8.0.0"},"framework": {"name": "Express","version": "1.2.3"},"name": "elastic-node","version": "3.14.0"}},"user": {"id": "123user", "username": "bar", "email": "bar@user.com"}, "labels": {"tag0": null, "tag1": "one", "tag2": 2}, "process": {"pid": 1234,"ppid": 6789,"title": "node","argv": ["node","server.js"]},"system": {"hostname": "prod1.example.com","architecture": "x64","platform": "darwin", "container": {"id": "container-id"}, "kubernetes": {"namespace": "namespace1", "pod": {"uid": "pod-uid", "name": "pod-name"}, "node": {"name": "node-name"}}},"cloud":{"account":{"id":"account_id","name":"account_name"},"availability_zone":"cloud_availability_zone","instance":{"id":"instance_id","name":"instance_name"},"machine":{"type":"machine_type"},"project":{"id":"project_id","name":"project_name"},"provider":"cloud_provider","region":"cloud_region","service":{"name":"lambda"} {"transaction": { "id": "945254c567a5417e", "trace_id": "0123456789abcdef0123456789abcdef", "parent_id": "abcdefabcdef01234567", "type": "request", "duration": 32.592981, "span_count": { "started": 43 }}} {"transaction": {"id": "4340a8e0df1906ecbfa9", "trace_id": "0acd456789abcdef0123456789abcdef", "name": "GET /api/types","type": "request","duration": 32.592981,"outcome":"success", "result": "success", "timestamp": 1496170407154000, "sampled": true, "span_count": {"started": 17},"context": {"service": {"runtime": {"version": "7.0"}},"page":{"referer":"http://localhost:8000/test/e2e/","url":"http://localhost:8000/test/e2e/general-usecase/"}, "request": {"socket": {"remote_address": "12.53.12.1","encrypted": true},"http_version": "1.1","method": "POST","url": {"protocol": "https:","full": "https://www.example.com/p/a/t/h?query=string#hash","hostname": "www.example.com","port": "8080","pathname": "/p/a/t/h","search": "?query=string","hash": "#hash","raw": "/p/a/t/h?query=string#hash"},"headers": {"user-agent":["Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36","Mozilla Chrome Edge"],"content-type": "text/html","cookie": "c1=v1, c2=v2","some-other-header": "foo","array": ["foo","bar","baz"]},"cookies": {"c1": "v1","c2": "v2"},"env": {"SERVER_SOFTWARE": "nginx","GATEWAY_INTERFACE": "CGI/1.1"},"body": {"str": "hello world","additional": { "foo": {},"bar": 123,"req": "additional information"}}},"response": {"status_code": 200,"headers": {"content-type": "application/json"},"headers_sent": true,"finished": true,"transfer_size":25.8,"encoded_body_size":26.90,"decoded_body_size":29.90}, "user": {"domain": "ldap://abc","id": "99","username": "foo"},"tags": {"organization_uuid": "9f0e9d64-c185-4d21-a6f4-4673ed561ec8", "tag2": 12, "tag3": 12.45, "tag4": false, "tag5": null },"custom": {"my_key": 1,"some_other_value": "foo bar","and_objects": {"foo": ["bar","baz"]},"(": "not a valid regex and that is fine"}}}} {"transaction": { "id": "cdef4340a8e0df19", "trace_id": "0acd456789abcdef0123456789abcdef", "type": "request", "duration": 13.980558, "timestamp": 1532976822281000, "sampled": null, "span_count": { "dropped": 55, "started": 436 }, "marks": {"navigationTiming": {"appBeforeBootstrap": 608.9300000000001,"navigationStart": -21},"another_mark": {"some_long": 10,"some_float": 10.0}, "performance": {}}, "context": { "request": { "socket": { "remote_address": "192.0.1", "encrypted": null }, "method": "POST", "headers": { "user-agent": null, "content-type": null, "cookie": null }, "url": { "protocol": null, "full": null, "hostname": null, "port": null, "pathname": null, "search": null, "hash": null, "raw": null } }, "response": { "headers": { "content-type": null } }, "service": {"environment":"testing","name": "service1","node": {"configured_name": "node-ABC"}, "language": {"version": "2.5", "name": "ruby"}, "agent": {"version": "2.2", "name": "elastic-ruby", "ephemeral_id": "justanid"}, "framework": {"version": "5.0", "name": "Rails"}, "version": "2", "runtime": {"version": "2.5", "name": "cruby"}}},"experience":{"cls":1,"fid":2.0,"tbt":3.4,"longtask":{"count":3,"sum":2.5,"max":1}}}} @@ -47,17 +49,8 @@ func Test_processMetadata(t *testing.T) { badAgentData := AgentData{Data: badBenchBody, ContentEncoding: ""} var mc MetadataContainer - if mc.Metadata == nil { - if err := ProcessMetadata(agentData, &mc); err != nil { - t.Fail() - } - } - - if mc.Metadata == nil { - if err := ProcessMetadata(badAgentData, &mc); err != nil { - t.Fail() - } - } + require.Error(t, ProcessMetadata(badAgentData, &mc)) + require.NoError(t, ProcessMetadata(agentData, &mc)) // Metadata is extracted as is. The agent name and version are replaced by values related to the extension in // logsapi.ProcessMetrics diff --git a/apm-lambda-extension/logsapi/process_metrics_test.go b/apm-lambda-extension/logsapi/process_metrics_test.go index e5abf5ad..309f3c50 100644 --- a/apm-lambda-extension/logsapi/process_metrics_test.go +++ b/apm-lambda-extension/logsapi/process_metrics_test.go @@ -27,7 +27,8 @@ import ( "elastic/apm-lambda-extension/extension" "elastic/apm-lambda-extension/model" - "gotest.tools/assert" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func Test_processPlatformReport(t *testing.T) { @@ -80,16 +81,12 @@ func Test_processPlatformReport(t *testing.T) { Runtime: model.Runtime{Name: os.Getenv("AWS_EXECUTION_ENV")}, Framework: model.Framework{Name: "AWS Lambda"}, } - mc.Metadata.Process = model.Process{} - mc.Metadata.System = model.System{} rawBytes, err := ProcessPlatformReport(context.Background(), &mc, &event, logEvent) - if err != nil { - t.Fail() - } + require.NoError(t, err) + requestBytes, err := extension.GetUncompressedBytes(rawBytes.Data, "") - if err != nil { - t.Fail() - } - assert.Equal(t, string(requestBytes), desiredOutput) + require.NoError(t, err) + + assert.Equal(t, desiredOutput, string(requestBytes)) } From c018cb69943977e5ed675c522394b306e0ae84c3 Mon Sep 17 00:00:00 2001 From: Jean-Louis Voiseux <48380853+jlvoiseux@users.noreply.github.com> Date: Fri, 3 Jun 2022 17:32:26 +0200 Subject: [PATCH 12/21] Fix: Use standard APM data model --- apm-lambda-extension/go.mod | 10 + apm-lambda-extension/go.sum | 39 +++ .../logsapi/process_metrics.go | 63 +++-- .../logsapi/process_metrics_test.go | 30 +-- apm-lambda-extension/model/apm_common.go | 20 -- apm-lambda-extension/model/apm_metadata.go | 245 ------------------ apm-lambda-extension/model/apm_metrics.go | 75 ------ 7 files changed, 95 insertions(+), 387 deletions(-) delete mode 100644 apm-lambda-extension/model/apm_common.go delete mode 100644 apm-lambda-extension/model/apm_metadata.go delete mode 100644 apm-lambda-extension/model/apm_metrics.go diff --git a/apm-lambda-extension/go.mod b/apm-lambda-extension/go.mod index f36283cc..452fb283 100644 --- a/apm-lambda-extension/go.mod +++ b/apm-lambda-extension/go.mod @@ -15,9 +15,19 @@ require ( gotest.tools v2.2.0+incompatible ) +require ( + github.com/tidwall/sjson v1.2.4 + go.elastic.co/apm v1.15.0 + go.elastic.co/fastjson v1.1.0 +) + require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/google/go-cmp v0.5.6 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/santhosh-tekuri/jsonschema v1.2.4 // indirect + github.com/tidwall/gjson v1.12.1 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) diff --git a/apm-lambda-extension/go.sum b/apm-lambda-extension/go.sum index a15011b3..fd32a941 100644 --- a/apm-lambda-extension/go.sum +++ b/apm-lambda-extension/go.sum @@ -1,12 +1,20 @@ +github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/elastic/go-licenser v0.3.1/go.mod h1:D8eNQk70FOCVBl3smCGQt/lv7meBeQno2eI1S5apiHQ= +github.com/elastic/go-sysinfo v1.1.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= +github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jcchavezs/porto v0.1.0/go.mod h1:fESH0gzDHiutHRdX2hv27ojnOVFco37hg1W6E9EZF4A= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg= github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= @@ -17,18 +25,36 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/magefile/mage v1.9.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magefile/mage v1.13.0 h1:XtLJl8bcCM7EFoO8FyH8XK3t7G5hQAeK+i4tq+veT9M= github.com/magefile/mage v1.13.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/procfs v0.0.0-20190425082905-87a4384529e0/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/santhosh-tekuri/jsonschema v1.2.4 h1:hNhW8e7t+H1vgY+1QeEQpveR6D4+OwKPXCfD2aieJis= +github.com/santhosh-tekuri/jsonschema v1.2.4/go.mod h1:TEAUOeZSmIxTTuHatJzrvARHiuO9LYd+cIxzgEHCQI4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tidwall/gjson v1.12.1 h1:ikuZsLdhr8Ws0IdROXUS1Gi4v9Z4pGqpX/CvJkxvfpo= +github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.4 h1:cuiLzLnaMeBhRmEv00Lpk3tkYrcxpmbU81tAY4Dw0tc= +github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp19pSNM= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.elastic.co/apm v1.15.0 h1:uPk2g/whK7c7XiZyz/YCUnAUBNPiyNeE3ARX3G6Gx7Q= +go.elastic.co/apm v1.15.0/go.mod h1:dylGv2HKR0tiCV+wliJz1KHtDyuD8SPe69oV7VyK6WY= go.elastic.co/ecszap v1.0.1 h1:mBxqEJAEXBlpi5+scXdzL7LTFGogbuxipJC0KTZicyA= go.elastic.co/ecszap v1.0.1/go.mod h1:SVjazT+QgNeHSGOCUHvRgN+ZRj5FkB7IXQQsncdF57A= +go.elastic.co/fastjson v1.1.0 h1:3MrGBWWVIxe/xvsbpghtkFoPciPhOCmjsR/HfwEeQR4= +go.elastic.co/fastjson v1.1.0/go.mod h1:boNGISWMjQsUPy/t6yqt2/1Wx4YNPSe+mZjlyw9vKKI= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -42,15 +68,24 @@ go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191025021431-6c3a3bfe00ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -60,6 +95,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200509030707-2212a7e161a5/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -69,6 +106,7 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -76,3 +114,4 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= diff --git a/apm-lambda-extension/logsapi/process_metrics.go b/apm-lambda-extension/logsapi/process_metrics.go index 32e8184d..e1fcca89 100644 --- a/apm-lambda-extension/logsapi/process_metrics.go +++ b/apm-lambda-extension/logsapi/process_metrics.go @@ -19,11 +19,13 @@ package logsapi import ( "context" - "encoding/json" "math" + "strconv" "elastic/apm-lambda-extension/extension" - "elastic/apm-lambda-extension/model" + + "go.elastic.co/apm/model" + "go.elastic.co/fastjson" ) type PlatformMetrics struct { @@ -40,7 +42,7 @@ type MetricsContainer struct { // Add adds a metric with the given name, labels, and value, // The labels are expected to be sorted lexicographically. -func (mc MetricsContainer) Add(name string, labels []model.MetricLabel, value float64) { +func (mc MetricsContainer) Add(name string, value float64) { mc.addMetric(name, model.Metric{Value: value}) } @@ -53,6 +55,15 @@ func (mc MetricsContainer) addMetric(name string, metric model.Metric) { mc.Metrics.Samples[name] = metric } +func (mc MetricsContainer) MarshalFastJSON(json *fastjson.Writer) error { + json.RawString(`{"metricset":`) + if err := mc.Metrics.MarshalFastJSON(json); err != nil { + return err + } + json.RawString(`}`) + return nil +} + func ProcessPlatformReport(ctx context.Context, metadataContainer *extension.MetadataContainer, functionData *extension.NextEventResponse, platformReport LogEvent) (extension.AgentData, error) { var metricsData []byte metricsContainer := MetricsContainer{ @@ -63,49 +74,37 @@ func ProcessPlatformReport(ctx context.Context, metadataContainer *extension.Met // APM Spec Fields // Timestamp - metricsContainer.Metrics.Timestamp = platformReport.Time.UnixMicro() + metricsContainer.Metrics.Timestamp = model.Time(platformReport.Time) // FaaS Fields - metricsContainer.Metrics.Labels = make(map[string]interface{}) - metricsContainer.Metrics.Labels["faas.execution"] = platformReport.Record.RequestId - metricsContainer.Metrics.Labels["faas.id"] = functionData.InvokedFunctionArn - if platformReportMetrics.InitDurationMs > 0 { - metricsContainer.Metrics.Labels["faas.coldstart"] = true - } else { - metricsContainer.Metrics.Labels["faas.coldstart"] = false + metricsContainer.Metrics.Labels = model.StringMap{ + {Key: "faas.execution", Value: platformReport.Record.RequestId}, + {Key: "faas.id", Value: functionData.InvokedFunctionArn}, + {Key: "faas.coldstart", Value: strconv.FormatBool(platformReportMetrics.InitDurationMs > 0)}, } // System - metricsContainer.Add("system.memory.total", nil, float64(platformReportMetrics.MemorySizeMB)*convMB2Bytes) // Unit : Bytes - metricsContainer.Add("system.memory.actual.free", nil, float64(platformReportMetrics.MemorySizeMB-platformReportMetrics.MaxMemoryUsedMB)*convMB2Bytes) // Unit : Bytes + // AWS uses binary multiples to compute memory : https://aws.amazon.com/about-aws/whats-new/2020/12/aws-lambda-supports-10gb-memory-6-vcpu-cores-lambda-functions/ + metricsContainer.Add("system.memory.total", float64(platformReportMetrics.MemorySizeMB)*convMB2Bytes) // Unit : Bytes + metricsContainer.Add("system.memory.actual.free", float64(platformReportMetrics.MemorySizeMB-platformReportMetrics.MaxMemoryUsedMB)*convMB2Bytes) // Unit : Bytes // Raw Metrics - // AWS uses binary multiples to compute memory : https://aws.amazon.com/about-aws/whats-new/2020/12/aws-lambda-supports-10gb-memory-6-vcpu-cores-lambda-functions/ - metricsContainer.Add("aws.lambda.metrics.TotalMemory", nil, float64(platformReportMetrics.MemorySizeMB)*convMB2Bytes) // Unit : Bytes - metricsContainer.Add("aws.lambda.metrics.UsedMemory", nil, float64(platformReportMetrics.MaxMemoryUsedMB)*convMB2Bytes) // Unit : Bytes - metricsContainer.Add("aws.lambda.metrics.Duration", nil, float64(platformReportMetrics.DurationMs)) // Unit : Milliseconds - metricsContainer.Add("aws.lambda.metrics.BilledDuration", nil, float64(platformReportMetrics.BilledDurationMs)) // Unit : Milliseconds - metricsContainer.Add("aws.lambda.metrics.ColdStartDuration", nil, float64(platformReportMetrics.InitDurationMs)) // Unit : Milliseconds + metricsContainer.Add("aws.lambda.metrics.Duration", float64(platformReportMetrics.DurationMs)) // Unit : Milliseconds + metricsContainer.Add("aws.lambda.metrics.BilledDuration", float64(platformReportMetrics.BilledDurationMs)) // Unit : Milliseconds + metricsContainer.Add("aws.lambda.metrics.ColdStartDuration", float64(platformReportMetrics.InitDurationMs)) // Unit : Milliseconds // In AWS Lambda, the Timeout is configured as an integer number of seconds. We use this assumption to derive the Timeout from // - The epoch corresponding to the end of the current invocation (its "deadline") // - The epoch corresponding to the start of the current invocation // - The multiplication / division then rounds the value to obtain a number of ms that can be expressed a multiple of 1000 (see initial assumption) - metricsContainer.Add("aws.lambda.metrics.Timeout", nil, math.Ceil(float64(functionData.DeadlineMs-functionData.Timestamp.UnixMilli())/1e3)*1e3) // Unit : Milliseconds + metricsContainer.Add("aws.lambda.metrics.Timeout", math.Ceil(float64(functionData.DeadlineMs-functionData.Timestamp.UnixMilli())/1e3)*1e3) // Unit : Milliseconds - metricsJson, err := json.Marshal(metricsContainer) - if err != nil { - return extension.AgentData{}, err + var jsonWriter fastjson.Writer + if err := metricsContainer.MarshalFastJSON(&jsonWriter); err != nil { + return extension.AgentData{Data: metricsData}, nil } if metadataContainer.Metadata != nil { - //TODO : Discuss relevance of displaying extension name - metadataContainer.Metadata.Service.Agent.Name = "apm-lambda-extension" - metadataContainer.Metadata.Service.Agent.Version = extension.Version - metadataJson, err := json.Marshal(metadataContainer) - if err != nil { - return extension.AgentData{}, err - } - metricsData = append(metadataJson, []byte("\n")...) + metricsData = append(metadataContainer.Metadata, []byte("\n")...) } - metricsData = append(metricsData, metricsJson...) + metricsData = append(metricsData, jsonWriter.Bytes()...) return extension.AgentData{Data: metricsData}, nil } diff --git a/apm-lambda-extension/logsapi/process_metrics_test.go b/apm-lambda-extension/logsapi/process_metrics_test.go index 309f3c50..3a411275 100644 --- a/apm-lambda-extension/logsapi/process_metrics_test.go +++ b/apm-lambda-extension/logsapi/process_metrics_test.go @@ -20,12 +20,12 @@ package logsapi import ( "context" "fmt" - "os" + "log" + "strings" "testing" "time" "elastic/apm-lambda-extension/extension" - "elastic/apm-lambda-extension/model" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -33,6 +33,10 @@ import ( func Test_processPlatformReport(t *testing.T) { + mc := extension.MetadataContainer{ + Metadata: []byte(fmt.Sprintf(`{"metadata":{"service":{"agent":{"name":"apm-lambda-extension","version":"%s"},"framework":{"name":"AWS Lambda","version":""},"language":{"name":"python","version":"3.9.8"},"runtime":{"name":"","version":""},"node":{}},"user":{},"process":{"pid":0},"system":{"container":{"id":""},"kubernetes":{"node":{},"pod":{}}},"cloud":{"provider":"","instance":{},"machine":{},"account":{},"project":{},"service":{}}}}`, extension.Version)), + } + timestamp := time.Now() pm := PlatformMetrics{ @@ -68,19 +72,9 @@ func Test_processPlatformReport(t *testing.T) { }, } - desiredOutput := fmt.Sprintf(`{"metadata":{"service":{"agent":{"name":"apm-lambda-extension","version":"%s"},"framework":{"name":"AWS Lambda","version":""},"language":{"name":"python","version":"3.9.8"},"runtime":{"name":"","version":""},"node":{}},"user":{},"process":{"pid":0},"system":{"container":{"id":""},"kubernetes":{"node":{},"pod":{}}},"cloud":{"provider":"","instance":{},"machine":{},"account":{},"project":{},"service":{}}}} -{"metricset":{"timestamp":%d,"transaction":{},"span":{},"tags":{"faas.coldstart":true,"faas.execution":"6f7f0961f83442118a7af6fe80b88d56","faas.id":"arn:aws:lambda:us-east-2:123456789012:function:custom-runtime"},"samples":{"aws.lambda.metrics.BilledDuration":{"value":183},"aws.lambda.metrics.ColdStartDuration":{"value":422.9700012207031},"aws.lambda.metrics.Duration":{"value":182.42999267578125},"aws.lambda.metrics.Timeout":{"value":5000},"aws.lambda.metrics.TotalMemory":{"value":134217728},"aws.lambda.metrics.UsedMemory":{"value":79691776},"system.memory.actual.free":{"value":54525952},"system.memory.total":{"value":134217728}}}}`, extension.Version, timestamp.UnixNano()/1e3) + desiredOutputMetadata := fmt.Sprintf(`{"metadata":{"service":{"agent":{"name":"apm-lambda-extension","version":"%s"},"framework":{"name":"AWS Lambda","version":""},"language":{"name":"python","version":"3.9.8"},"runtime":{"name":"","version":""},"node":{}},"user":{},"process":{"pid":0},"system":{"container":{"id":""},"kubernetes":{"node":{},"pod":{}}},"cloud":{"provider":"","instance":{},"machine":{},"account":{},"project":{},"service":{}}}}`, extension.Version) - mc := extension.MetadataContainer{ - Metadata: &model.Metadata{}, - } - mc.Metadata.Service = model.Service{ - Name: os.Getenv("AWS_LAMBDA_FUNCTION_NAME"), - Agent: model.Agent{Name: "python", Version: "6.7.2"}, - Language: model.Language{Name: "python", Version: "3.9.8"}, - Runtime: model.Runtime{Name: os.Getenv("AWS_EXECUTION_ENV")}, - Framework: model.Framework{Name: "AWS Lambda"}, - } + desiredOutputMetrics := fmt.Sprintf(`{"metricset":{"samples":{"aws.lambda.metrics.BilledDuration":{"value":183},"aws.lambda.metrics.ColdStartDuration":{"value":422.9700012207031},"aws.lambda.metrics.Timeout":{"value":5000},"system.memory.total":{"value":1.34217728e+08},"system.memory.actual.free":{"value":5.4525952e+07},"aws.lambda.metrics.Duration":{"value":182.42999267578125}},"timestamp":%d,"tags":{"faas.execution":"6f7f0961f83442118a7af6fe80b88d56","faas.id":"arn:aws:lambda:us-east-2:123456789012:function:custom-runtime","faas.coldstart":"true"}}}`, timestamp.UnixNano()/1e3) rawBytes, err := ProcessPlatformReport(context.Background(), &mc, &event, logEvent) require.NoError(t, err) @@ -88,5 +82,11 @@ func Test_processPlatformReport(t *testing.T) { requestBytes, err := extension.GetUncompressedBytes(rawBytes.Data, "") require.NoError(t, err) - assert.Equal(t, desiredOutput, string(requestBytes)) + out := string(requestBytes) + log.Println(out) + + processingResult := strings.Split(string(requestBytes), "\n") + + assert.JSONEq(t, desiredOutputMetadata, processingResult[0]) + assert.JSONEq(t, desiredOutputMetrics, processingResult[1]) } diff --git a/apm-lambda-extension/model/apm_common.go b/apm-lambda-extension/model/apm_common.go deleted file mode 100644 index 35f3cc46..00000000 --- a/apm-lambda-extension/model/apm_common.go +++ /dev/null @@ -1,20 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package model - -type StringMap map[string]interface{} diff --git a/apm-lambda-extension/model/apm_metadata.go b/apm-lambda-extension/model/apm_metadata.go deleted file mode 100644 index 2f12f5ef..00000000 --- a/apm-lambda-extension/model/apm_metadata.go +++ /dev/null @@ -1,245 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package model - -// Service represents the service handling transactions being traced. -type Service struct { - // Name is the immutable name of the service. - Name string `json:"name,omitempty"` - - // Version is the version of the service, if it has one. - Version string `json:"version,omitempty"` - - // Environment is the name of the service's environment, if it has - // one, e.g. "production" or "staging". - Environment string `json:"environment,omitempty"` - - // Agent holds information about the Elastic APM agent tracing this - // service's transactions. - Agent Agent `json:"agent,omitempty"` - - // Framework holds information about the service's framework, if any. - Framework Framework `json:"framework,omitempty"` - - // Language holds information about the programming language in which - // the service is written. - Language Language `json:"language,omitempty"` - - // Runtime holds information about the programming language runtime - // running this service. - Runtime Runtime `json:"runtime,omitempty"` - - // Node holds unique information about each service node - Node ServiceNode `json:"node,omitempty"` -} - -// Agent holds information about the Elastic APM agent. -type Agent struct { - // Name is the name of the Elastic APM agent, e.g. "Go". - Name string `json:"name"` - - // Version is the version of the Elastic APM agent, e.g. "1.0.0". - Version string `json:"version"` -} - -// Framework holds information about the framework (typically web) -// used by the service. -type Framework struct { - // Name is the name of the framework. - Name string `json:"name"` - - // Version is the version of the framework. - Version string `json:"version"` -} - -// Language holds information about the programming language used. -type Language struct { - // Name is the name of the programming language. - Name string `json:"name"` - - // Version is the version of the programming language. - Version string `json:"version,omitempty"` -} - -// Runtime holds information about the programming language runtime. -type Runtime struct { - // Name is the name of the programming language runtime. - Name string `json:"name"` - - // Version is the version of the programming language runtime. - Version string `json:"version"` -} - -// ServiceNode holds unique information about each service node -type ServiceNode struct { - // ConfiguredName holds the name of the service node - ConfiguredName string `json:"configured_name,omitempty"` -} - -// System represents the system (operating system and machine) running the -// service. -type System struct { - // Architecture is the system's hardware architecture. - Architecture string `json:"architecture,omitempty"` - - // Hostname is the system's hostname. - Hostname string `json:"hostname,omitempty"` - - // Platform is the system's platform, or operating system name. - Platform string `json:"platform,omitempty"` - - // Container describes the container running the service. - Container Container `json:"container,omitempty"` - - // Kubernetes describes the kubernetes node and pod running the service. - Kubernetes Kubernetes `json:"kubernetes,omitempty"` -} - -// Process represents an operating system process. -type Process struct { - // Pid is the process ID. - Pid int `json:"pid"` - - // Ppid is the parent process ID, if known. - Ppid *int `json:"ppid,omitempty"` - - // Title is the title of the process. - Title string `json:"title,omitempty"` - - // Argv holds the command line arguments used to start the process. - Argv []string `json:"argv,omitempty"` -} - -// Container represents the container (e.g. Docker) running the service. -type Container struct { - // ID is the unique container ID. - ID string `json:"id"` -} - -// Kubernetes describes properties of the Kubernetes node and pod in which -// the service is running. -type Kubernetes struct { - // Namespace names the Kubernetes namespace in which the pod exists. - Namespace string `json:"namespace,omitempty"` - - // Node describes the Kubernetes node running the service's pod. - Node KubernetesNode `json:"node,omitempty"` - - // Pod describes the Kubernetes pod running the service. - Pod KubernetesPod `json:"pod,omitempty"` -} - -// KubernetesNode describes a Kubernetes node. -type KubernetesNode struct { - // Name holds the node name. - Name string `json:"name,omitempty"` -} - -// KubernetesPod describes a Kubernetes pod. -type KubernetesPod struct { - // Name holds the pod name. - Name string `json:"name,omitempty"` - - // UID holds the pod UID. - UID string `json:"uid,omitempty"` -} - -// Cloud represents the cloud in which the service is running. -type Cloud struct { - // Provider is the cloud provider name, e.g. aws, azure, gcp. - Provider string `json:"provider"` - - // Region is the cloud region name, e.g. us-east-1. - Region string `json:"region,omitempty"` - - // AvailabilityZone is the cloud availability zone name, e.g. us-east-1a. - AvailabilityZone string `json:"availability_zone,omitempty"` - - // Instance holds information about the cloud instance (virtual machine). - Instance CloudInstance `json:"instance,omitempty"` - - // Machine also holds information about the cloud instance (virtual machine). - Machine CloudMachine `json:"machine,omitempty"` - - // Account holds information about the cloud account. - Account CloudAccount `json:"account,omitempty"` - - // Project holds information about the cloud project. - Project CloudProject `json:"project,omitempty"` - - Service CloudService `json:"service,omitempty"` -} - -// CloudInstance holds information about a cloud instance (virtual machine). -type CloudInstance struct { - // ID holds the cloud instance identifier. - ID string `json:"id,omitempty"` - - // ID holds the cloud instance name. - Name string `json:"name,omitempty"` -} - -// CloudMachine holds information about a cloud instance (virtual machine). -type CloudMachine struct { - // Type holds the cloud instance type, e.g. t2.medium. - Type string `json:"type,omitempty"` -} - -// CloudAccount holds information about a cloud account. -type CloudAccount struct { - // ID holds the cloud account identifier. - ID string `json:"id,omitempty"` - - // ID holds the cloud account name. - Name string `json:"name,omitempty"` -} - -// CloudProject holds information about a cloud project. -type CloudProject struct { - // ID holds the cloud project identifier. - ID string `json:"id,omitempty"` - - // Name holds the cloud project name. - Name string `json:"name,omitempty"` -} - -type CloudService struct { - Name string `json:"name,omitempty"` -} - -// User holds information about an authenticated user. -type User struct { - // Username holds the username of the user. - Username string `json:"username,omitempty"` - - // ID identifies the user, e.g. a primary key. This may be - // a string or number. - ID string `json:"id,omitempty"` - - // Email holds the email address of the user. - Email string `json:"email,omitempty"` -} - -type Metadata struct { - Service Service `json:"service,omitempty"` - User User `json:"user,omitempty"` - Labels map[string]interface{} `json:"labels,omitempty"` - Process Process `json:"process,omitempty"` - System System `json:"system,omitempty"` - Cloud Cloud `json:"cloud,omitempty"` -} diff --git a/apm-lambda-extension/model/apm_metrics.go b/apm-lambda-extension/model/apm_metrics.go deleted file mode 100644 index 8b8b7a62..00000000 --- a/apm-lambda-extension/model/apm_metrics.go +++ /dev/null @@ -1,75 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package model - -// MetricLabel is a name/value pair for labeling metrics. -type MetricLabel struct { - // Name is the label name. - Name string - - // Value is the label value. - Value string -} - -// MetricsTransaction holds transaction identifiers for metrics. -type MetricsTransaction struct { - Type string `json:"type,omitempty"` - Name string `json:"name,omitempty"` -} - -// MetricsSpan holds span identifiers for metrics. -type MetricsSpan struct { - Type string `json:"type,omitempty"` - Subtype string `json:"subtype,omitempty"` -} - -// Metric holds metric values. -type Metric struct { - Type string `json:"type,omitempty"` - // Value holds the metric value. - Value float64 `json:"value"` - // Buckets holds the metric bucket values. - Values []float64 `json:"values,omitempty"` - // Count holds the metric observation count for the bucket. - Counts []uint64 `json:"counts,omitempty"` -} - -// Metrics holds a set of metric samples, with an optional set of labels. -type Metrics struct { - // Timestamp holds the time at which the metric samples were taken. - Timestamp int64 `json:"timestamp"` - - // Transaction optionally holds the name and type of transactions - // with which these metrics are associated. - Transaction MetricsTransaction `json:"transaction,omitempty"` - - // Span optionally holds the type and subtype of the spans with - // which these metrics are associated. - Span MetricsSpan `json:"span,omitempty"` - - // Labels holds a set of labels associated with the metrics. - // The labels apply uniformly to all metric samples in the set. - // - // NOTE(axw) the schema calls the field "tags", but we use - // "labels" for agent-internal consistency. Labels aligns better - // with the common schema, anyway. - Labels StringMap `json:"tags,omitempty"` - - // Samples holds a map of metric samples, keyed by metric name. - Samples map[string]Metric `json:"samples"` -} From fd5ce97eaa9ccdeaf9c7dc49b1f0066fbd58c785 Mon Sep 17 00:00:00 2001 From: Jean-Louis Voiseux <48380853+jlvoiseux@users.noreply.github.com> Date: Fri, 3 Jun 2022 17:33:37 +0200 Subject: [PATCH 13/21] Fix: remove metadataJSON encoding / decoding --- .../extension/apm_server_transport.go | 3 +- .../extension/process_metadata.go | 33 +++++++------------ .../extension/process_metadata_test.go | 28 ++++------------ 3 files changed, 21 insertions(+), 43 deletions(-) diff --git a/apm-lambda-extension/extension/apm_server_transport.go b/apm-lambda-extension/extension/apm_server_transport.go index 53a29984..326eb2b5 100644 --- a/apm-lambda-extension/extension/apm_server_transport.go +++ b/apm-lambda-extension/extension/apm_server_transport.go @@ -86,10 +86,11 @@ func (transport *ApmServerTransport) ForwardApmData(ctx context.Context, metadat return nil case agentData := <-transport.dataChannel: if metadataContainer.Metadata == nil { - err := ProcessMetadata(agentData, metadataContainer) + metadata, err := ProcessMetadata(agentData) if err != nil { Log.Errorf("Error extracting metadata from agent payload %v", err) } + metadataContainer.Metadata = metadata } if err := transport.PostToApmServer(ctx, agentData); err != nil { return fmt.Errorf("error sending to APM server, skipping: %v", err) diff --git a/apm-lambda-extension/extension/process_metadata.go b/apm-lambda-extension/extension/process_metadata.go index 27f757a4..b23eee46 100644 --- a/apm-lambda-extension/extension/process_metadata.go +++ b/apm-lambda-extension/extension/process_metadata.go @@ -18,44 +18,35 @@ package extension import ( + "bufio" "bytes" "compress/gzip" "compress/zlib" - "encoding/json" "fmt" - "io" "io/ioutil" - - "elastic/apm-lambda-extension/model" + "strings" "github.com/pkg/errors" ) type MetadataContainer struct { - Metadata *model.Metadata `json:"metadata"` + Metadata []byte } -func ProcessMetadata(data AgentData, container *MetadataContainer) error { +// ProcessMetadata return a byte array containing the Metadata marshaled in JSON +// In case we want to update the Metadata values, usage of https://github.com/tidwall/sjson is advised +func ProcessMetadata(data AgentData) ([]byte, error) { uncompressedData, err := GetUncompressedBytes(data.Data, data.ContentEncoding) if err != nil { - return errors.New(fmt.Sprintf("Error uncompressing agent data for metadata extraction : %v", err)) + return nil, errors.New(fmt.Sprintf("Error uncompressing agent data for metadata extraction : %v", err)) } - decoder := json.NewDecoder(bytes.NewReader(uncompressedData)) - for { - err = decoder.Decode(container) - if container.Metadata != nil { - Log.Debug("Metadata decoded") - break - } - if err != nil { - if err == io.EOF { - return errors.New("No metadata in current agent transaction") - } else { - return errors.New(fmt.Sprintf("Error uncompressing agent data for metadata extraction : %v", err)) - } + scanner := bufio.NewScanner(strings.NewReader(string(uncompressedData))) + for scanner.Scan() { + if strings.Contains(strings.ToLower(scanner.Text()), "metadata") { + return scanner.Bytes(), nil } } - return nil + return nil, errors.New("No metadata found in APM agent payload") } func GetUncompressedBytes(rawBytes []byte, encodingType string) ([]byte, error) { diff --git a/apm-lambda-extension/extension/process_metadata_test.go b/apm-lambda-extension/extension/process_metadata_test.go index 4700f5a5..f9ee03ad 100644 --- a/apm-lambda-extension/extension/process_metadata_test.go +++ b/apm-lambda-extension/extension/process_metadata_test.go @@ -18,11 +18,10 @@ package extension import ( - "encoding/json" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "gotest.tools/assert" ) func Test_processMetadata(t *testing.T) { @@ -34,30 +33,17 @@ func Test_processMetadata(t *testing.T) { {"transaction": { "id": "cdef4340a8e0df19", "trace_id": "0acd456789abcdef0123456789abcdef", "type": "request", "duration": 13.980558, "timestamp": 1532976822281000, "sampled": null, "span_count": { "dropped": 55, "started": 436 }, "marks": {"navigationTiming": {"appBeforeBootstrap": 608.9300000000001,"navigationStart": -21},"another_mark": {"some_long": 10,"some_float": 10.0}, "performance": {}}, "context": { "request": { "socket": { "remote_address": "192.0.1", "encrypted": null }, "method": "POST", "headers": { "user-agent": null, "content-type": null, "cookie": null }, "url": { "protocol": null, "full": null, "hostname": null, "port": null, "pathname": null, "search": null, "hash": null, "raw": null } }, "response": { "headers": { "content-type": null } }, "service": {"environment":"testing","name": "service1","node": {"configured_name": "node-ABC"}, "language": {"version": "2.5", "name": "ruby"}, "agent": {"version": "2.2", "name": "elastic-ruby", "ephemeral_id": "justanid"}, "framework": {"version": "5.0", "name": "Rails"}, "version": "2", "runtime": {"version": "2.5", "name": "cruby"}}},"experience":{"cls":1,"fid":2.0,"tbt":3.4,"longtask":{"count":3,"sum":2.5,"max":1}}}} {"transaction": { "id": "00xxxxFFaaaa1234", "trace_id": "0123456789abcdef0123456789abcdef", "name": "amqp receive", "parent_id": "abcdefabcdef01234567", "type": "messaging", "duration": 3, "span_count": { "started": 1 }, "context": {"message": {"queue": { "name": "new_users"}, "age":{ "ms": 1577958057123}, "headers": {"user_id": "1ax3", "involved_services": ["user", "auth"]}, "body": "user created", "routing_key": "user-created-transaction"}},"session":{"id":"sunday","sequence":123}}} {"transaction": { "name": "july-2021-delete-after-july-31", "type": "lambda", "result": "success", "id": "142e61450efb8574", "trace_id": "eb56529a1f461c5e7e2f66ecb075e983", "subtype": null, "action": null, "duration": 38.853, "timestamp": 1631736666365048, "sampled": true, "context": { "cloud": { "origin": { "account": { "id": "abc123" }, "provider": "aws", "region": "us-east-1", "service": { "name": "serviceName" } } }, "service": { "origin": { "id": "abc123", "name": "service-name", "version": "1.0" } }, "user": {}, "tags": {}, "custom": { } }, "sync": true, "span_count": { "started": 0 }, "outcome": "unknown", "faas": { "coldstart": false, "execution": "2e13b309-23e1-417f-8bf7-074fc96bc683", "trigger": { "request_id": "FuH2Cir_vHcEMUA=", "type": "http" } }, "sample_rate": 1 } } -`) - - // The number of closing curly braces does not match - this body is not supposed to be unmarshaled successfully, which is accordingly tested. - badBenchBody := []byte(`{"metadata": {"service": {"name": "BAD","node": {"configured_name": "node-123"},"version": "5.1.3","environment": "staging","language": {"name": "ecmascript","version": "8"},"runtime": {"name": "node","version": "8.0.0"},"framework": {"name": "Express","version": "1.2.3"},"name": "elastic-node","version": "3.14.0"}},"user": {"id": "123user", "username": "bar", "email": "bar@user.com"}, "labels": {"tag0": null, "tag1": "one", "tag2": 2}, "process": {"pid": 1234,"ppid": 6789,"title": "node","argv": ["node","server.js"]},"system": {"hostname": "prod1.example.com","architecture": "x64","platform": "darwin", "container": {"id": "container-id"}, "kubernetes": {"namespace": "namespace1", "pod": {"uid": "pod-uid", "name": "pod-name"}, "node": {"name": "node-name"}}},"cloud":{"account":{"id":"account_id","name":"account_name"},"availability_zone":"cloud_availability_zone","instance":{"id":"instance_id","name":"instance_name"},"machine":{"type":"machine_type"},"project":{"id":"project_id","name":"project_name"},"provider":"cloud_provider","region":"cloud_region","service":{"name":"lambda"} -{"transaction": { "id": "945254c567a5417e", "trace_id": "0123456789abcdef0123456789abcdef", "parent_id": "abcdefabcdef01234567", "type": "request", "duration": 32.592981, "span_count": { "started": 43 }}} -{"transaction": {"id": "4340a8e0df1906ecbfa9", "trace_id": "0acd456789abcdef0123456789abcdef", "name": "GET /api/types","type": "request","duration": 32.592981,"outcome":"success", "result": "success", "timestamp": 1496170407154000, "sampled": true, "span_count": {"started": 17},"context": {"service": {"runtime": {"version": "7.0"}},"page":{"referer":"http://localhost:8000/test/e2e/","url":"http://localhost:8000/test/e2e/general-usecase/"}, "request": {"socket": {"remote_address": "12.53.12.1","encrypted": true},"http_version": "1.1","method": "POST","url": {"protocol": "https:","full": "https://www.example.com/p/a/t/h?query=string#hash","hostname": "www.example.com","port": "8080","pathname": "/p/a/t/h","search": "?query=string","hash": "#hash","raw": "/p/a/t/h?query=string#hash"},"headers": {"user-agent":["Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36","Mozilla Chrome Edge"],"content-type": "text/html","cookie": "c1=v1, c2=v2","some-other-header": "foo","array": ["foo","bar","baz"]},"cookies": {"c1": "v1","c2": "v2"},"env": {"SERVER_SOFTWARE": "nginx","GATEWAY_INTERFACE": "CGI/1.1"},"body": {"str": "hello world","additional": { "foo": {},"bar": 123,"req": "additional information"}}},"response": {"status_code": 200,"headers": {"content-type": "application/json"},"headers_sent": true,"finished": true,"transfer_size":25.8,"encoded_body_size":26.90,"decoded_body_size":29.90}, "user": {"domain": "ldap://abc","id": "99","username": "foo"},"tags": {"organization_uuid": "9f0e9d64-c185-4d21-a6f4-4673ed561ec8", "tag2": 12, "tag3": 12.45, "tag4": false, "tag5": null },"custom": {"my_key": 1,"some_other_value": "foo bar","and_objects": {"foo": ["bar","baz"]},"(": "not a valid regex and that is fine"}}}} -{"transaction": { "id": "cdef4340a8e0df19", "trace_id": "0acd456789abcdef0123456789abcdef", "type": "request", "duration": 13.980558, "timestamp": 1532976822281000, "sampled": null, "span_count": { "dropped": 55, "started": 436 }, "marks": {"navigationTiming": {"appBeforeBootstrap": 608.9300000000001,"navigationStart": -21},"another_mark": {"some_long": 10,"some_float": 10.0}, "performance": {}}, "context": { "request": { "socket": { "remote_address": "192.0.1", "encrypted": null }, "method": "POST", "headers": { "user-agent": null, "content-type": null, "cookie": null }, "url": { "protocol": null, "full": null, "hostname": null, "port": null, "pathname": null, "search": null, "hash": null, "raw": null } }, "response": { "headers": { "content-type": null } }, "service": {"environment":"testing","name": "service1","node": {"configured_name": "node-ABC"}, "language": {"version": "2.5", "name": "ruby"}, "agent": {"version": "2.2", "name": "elastic-ruby", "ephemeral_id": "justanid"}, "framework": {"version": "5.0", "name": "Rails"}, "version": "2", "runtime": {"version": "2.5", "name": "cruby"}}},"experience":{"cls":1,"fid":2.0,"tbt":3.4,"longtask":{"count":3,"sum":2.5,"max":1}}}} -{"transaction": { "id": "00xxxxFFaaaa1234", "trace_id": "0123456789abcdef0123456789abcdef", "name": "amqp receive", "parent_id": "abcdefabcdef01234567", "type": "messaging", "duration": 3, "span_count": { "started": 1 }, "context": {"message": {"queue": { "name": "new_users"}, "age":{ "ms": 1577958057123}, "headers": {"user_id": "1ax3", "involved_services": ["user", "auth"]}, "body": "user created", "routing_key": "user-created-transaction"}},"session":{"id":"sunday","sequence":123}}} -{"transaction": { "name": "july-2021-delete-after-july-31", "type": "lambda", "result": "success", "id": "142e61450efb8574", "trace_id": "eb56529a1f461c5e7e2f66ecb075e983", "subtype": null, "action": null, "duration": 38.853, "timestamp": 1631736666365048, "sampled": true, "context": { "cloud": { "origin": { "account": { "id": "abc123" }, "provider": "aws", "region": "us-east-1", "service": { "name": "serviceName" } } }, "service": { "origin": { "id": "abc123", "name": "service-name", "version": "1.0" } }, "user": {}, "tags": {}, "custom": { } }, "sync": true, "span_count": { "started": 0 }, "outcome": "unknown", "faas": { "coldstart": false, "execution": "2e13b309-23e1-417f-8bf7-074fc96bc683", "trigger": { "request_id": "FuH2Cir_vHcEMUA=", "type": "http" } }, "sample_rate": 1 } } `) agentData := AgentData{Data: benchBody, ContentEncoding: ""} - badAgentData := AgentData{Data: badBenchBody, ContentEncoding: ""} - var mc MetadataContainer + extractedMetadata, err := ProcessMetadata(agentData) + require.NoError(t, err) - require.Error(t, ProcessMetadata(badAgentData, &mc)) - require.NoError(t, ProcessMetadata(agentData, &mc)) - - // Metadata is extracted as is. The agent name and version are replaced by values related to the extension in - // logsapi.ProcessMetrics + // Metadata is extracted as is. desiredMetadata := []byte(`{"metadata":{"service":{"name":"1234_service-12a3","version":"5.1.3","environment":"staging","agent":{"name":"elastic-node","version":"3.14.0"},"framework":{"name":"Express","version":"1.2.3"},"language":{"name":"ecmascript","version":"8"},"runtime":{"name":"node","version":"8.0.0"},"node":{"configured_name":"node-123"}},"user":{"username":"bar","id":"123user","email":"bar@user.com"},"labels":{"tag0":null,"tag1":"one","tag2":2},"process":{"pid":1234,"ppid":6789,"title":"node","argv":["node","server.js"]},"system":{"architecture":"x64","hostname":"prod1.example.com","platform":"darwin","container":{"id":"container-id"},"kubernetes":{"namespace":"namespace1","node":{"name":"node-name"},"pod":{"name":"pod-name","uid":"pod-uid"}}},"cloud":{"provider":"cloud_provider","region":"cloud_region","availability_zone":"cloud_availability_zone","instance":{"id":"instance_id","name":"instance_name"},"machine":{"type":"machine_type"},"account":{"id":"account_id","name":"account_name"},"project":{"id":"project_id","name":"project_name"},"service":{"name":"lambda"}}}}`) - extractedMetadata, err := json.Marshal(mc) if err != nil { - Log.Errorf("Could not marshal extracted metadata : %v", err) + Log.Errorf("Could not marshal extracted Metadata : %v", err) } - assert.DeepEqual(t, desiredMetadata, extractedMetadata) + + assert.JSONEq(t, string(desiredMetadata), string(extractedMetadata)) } From c058162845ef1bd3a8268a55543d58e62c753b3e Mon Sep 17 00:00:00 2001 From: Jean-Louis Voiseux <48380853+jlvoiseux@users.noreply.github.com> Date: Fri, 3 Jun 2022 17:33:47 +0200 Subject: [PATCH 14/21] E2E tests update --- apm-lambda-extension/main_test.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/apm-lambda-extension/main_test.go b/apm-lambda-extension/main_test.go index 8f3325d6..553baff6 100644 --- a/apm-lambda-extension/main_test.go +++ b/apm-lambda-extension/main_test.go @@ -598,8 +598,6 @@ func TestMetricsWithoutMetadata(t *testing.T) { assert.Contains(t, apmServerInternals.Data, `aws.lambda.metrics.BilledDuration":{"value":60`) assert.Contains(t, apmServerInternals.Data, `aws.lambda.metrics.Duration":{"value":59.9`) - assert.Contains(t, apmServerInternals.Data, `aws.lambda.metrics.TotalMemory":{"value":134217728`) - assert.Contains(t, apmServerInternals.Data, `aws.lambda.metrics.UsedMemory":{"value":62914560`) assert.Contains(t, apmServerInternals.Data, `aws.lambda.metrics.ColdStartDuration":{"value":500`) } @@ -610,7 +608,7 @@ func TestMetricsWithMetadata(t *testing.T) { eventsChannel := newTestStructs(t) apmServerInternals, _ := newMockApmServer(t) newMockLambdaServer(t, eventsChannel) - + eventsChain := []MockEvent{ {Type: InvokeStandardMetadata, APMServerBehavior: TimelyResponse, ExecutionDuration: 1, Timeout: 5}, {Type: InvokeStandardMetadata, APMServerBehavior: TimelyResponse, ExecutionDuration: 1, Timeout: 5}, @@ -618,10 +616,8 @@ func TestMetricsWithMetadata(t *testing.T) { eventQueueGenerator(eventsChain, eventsChannel) assert.NotPanics(t, main) - assert.Contains(t, apmServerInternals.Data, fmt.Sprintf(`{"metadata":{"service":{"name":"1234_service-12a3","version":"5.1.3","environment":"staging","agent":{"name":"apm-lambda-extension","version":"%s"},"framework":{"name":"Express","version":"1.2.3"},"language":{"name":"ecmascript","version":"8"},"runtime":{"name":"node","version":"8.0.0"},"node":{"configured_name":"node-123"}},"user":{"username":"bar","id":"123user","email":"bar@user.com"},"labels":{"tag0":null,"tag1":"one","tag2":2},"process":{"pid":1234,"ppid":6789,"title":"node","argv":["node","server.js"]},"system":{"architecture":"x64","hostname":"prod1.example.com","platform":"darwin","container":{"id":"container-id"},"kubernetes":{"namespace":"namespace1","node":{"name":"node-name"},"pod":{"name":"pod-name","uid":"pod-uid"}}},"cloud":{"provider":"cloud_provider","region":"cloud_region","availability_zone":"cloud_availability_zone","instance":{"id":"instance_id","name":"instance_name"},"machine":{"type":"machine_type"},"account":{"id":"account_id","name":"account_name"},"project":{"id":"project_id","name":"project_name"},"service":{"name":"lambda"}}}}`, extension.Version)) + assert.Contains(t, apmServerInternals.Data, `{"metadata":{"service":{"name":"1234_service-12a3","version":"5.1.3","environment":"staging","agent":{"name":"elastic-node","version":"3.14.0"},"framework":{"name":"Express","version":"1.2.3"},"language":{"name":"ecmascript","version":"8"},"runtime":{"name":"node","version":"8.0.0"},"node":{"configured_name":"node-123"}},"user":{"username":"bar","id":"123user","email":"bar@user.com"},"labels":{"tag0":null,"tag1":"one","tag2":2},"process":{"pid":1234,"ppid":6789,"title":"node","argv":["node","server.js"]},"system":{"architecture":"x64","hostname":"prod1.example.com","platform":"darwin","container":{"id":"container-id"},"kubernetes":{"namespace":"namespace1","node":{"name":"node-name"},"pod":{"name":"pod-name","uid":"pod-uid"}}},"cloud":{"provider":"cloud_provider","region":"cloud_region","availability_zone":"cloud_availability_zone","instance":{"id":"instance_id","name":"instance_name"},"machine":{"type":"machine_type"},"account":{"id":"account_id","name":"account_name"},"project":{"id":"project_id","name":"project_name"},"service":{"name":"lambda"}}}}`) assert.Contains(t, apmServerInternals.Data, `aws.lambda.metrics.BilledDuration":{"value":60`) assert.Contains(t, apmServerInternals.Data, `aws.lambda.metrics.Duration":{"value":59.9`) - assert.Contains(t, apmServerInternals.Data, `aws.lambda.metrics.TotalMemory":{"value":134217728`) - assert.Contains(t, apmServerInternals.Data, `aws.lambda.metrics.UsedMemory":{"value":62914560`) assert.Contains(t, apmServerInternals.Data, `aws.lambda.metrics.ColdStartDuration":{"value":500`) } From d5952d6e4378960ad21e8ca237f267fae05668eb Mon Sep 17 00:00:00 2001 From: Jean-Louis Voiseux <48380853+jlvoiseux@users.noreply.github.com> Date: Fri, 3 Jun 2022 17:44:10 +0200 Subject: [PATCH 15/21] Use lower_snake_case for metrics names --- apm-lambda-extension/logsapi/process_metrics.go | 8 ++++---- apm-lambda-extension/logsapi/process_metrics_test.go | 2 +- apm-lambda-extension/main_test.go | 12 ++++++------ 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/apm-lambda-extension/logsapi/process_metrics.go b/apm-lambda-extension/logsapi/process_metrics.go index e1fcca89..e0626af8 100644 --- a/apm-lambda-extension/logsapi/process_metrics.go +++ b/apm-lambda-extension/logsapi/process_metrics.go @@ -87,14 +87,14 @@ func ProcessPlatformReport(ctx context.Context, metadataContainer *extension.Met metricsContainer.Add("system.memory.actual.free", float64(platformReportMetrics.MemorySizeMB-platformReportMetrics.MaxMemoryUsedMB)*convMB2Bytes) // Unit : Bytes // Raw Metrics - metricsContainer.Add("aws.lambda.metrics.Duration", float64(platformReportMetrics.DurationMs)) // Unit : Milliseconds - metricsContainer.Add("aws.lambda.metrics.BilledDuration", float64(platformReportMetrics.BilledDurationMs)) // Unit : Milliseconds - metricsContainer.Add("aws.lambda.metrics.ColdStartDuration", float64(platformReportMetrics.InitDurationMs)) // Unit : Milliseconds + metricsContainer.Add("aws.lambda.metrics.duration", float64(platformReportMetrics.DurationMs)) // Unit : Milliseconds + metricsContainer.Add("aws.lambda.metrics.billed_duration", float64(platformReportMetrics.BilledDurationMs)) // Unit : Milliseconds + metricsContainer.Add("aws.lambda.metrics.cold_start_duration", float64(platformReportMetrics.InitDurationMs)) // Unit : Milliseconds // In AWS Lambda, the Timeout is configured as an integer number of seconds. We use this assumption to derive the Timeout from // - The epoch corresponding to the end of the current invocation (its "deadline") // - The epoch corresponding to the start of the current invocation // - The multiplication / division then rounds the value to obtain a number of ms that can be expressed a multiple of 1000 (see initial assumption) - metricsContainer.Add("aws.lambda.metrics.Timeout", math.Ceil(float64(functionData.DeadlineMs-functionData.Timestamp.UnixMilli())/1e3)*1e3) // Unit : Milliseconds + metricsContainer.Add("aws.lambda.metrics.timeout", math.Ceil(float64(functionData.DeadlineMs-functionData.Timestamp.UnixMilli())/1e3)*1e3) // Unit : Milliseconds var jsonWriter fastjson.Writer if err := metricsContainer.MarshalFastJSON(&jsonWriter); err != nil { diff --git a/apm-lambda-extension/logsapi/process_metrics_test.go b/apm-lambda-extension/logsapi/process_metrics_test.go index 3a411275..bb683c90 100644 --- a/apm-lambda-extension/logsapi/process_metrics_test.go +++ b/apm-lambda-extension/logsapi/process_metrics_test.go @@ -74,7 +74,7 @@ func Test_processPlatformReport(t *testing.T) { desiredOutputMetadata := fmt.Sprintf(`{"metadata":{"service":{"agent":{"name":"apm-lambda-extension","version":"%s"},"framework":{"name":"AWS Lambda","version":""},"language":{"name":"python","version":"3.9.8"},"runtime":{"name":"","version":""},"node":{}},"user":{},"process":{"pid":0},"system":{"container":{"id":""},"kubernetes":{"node":{},"pod":{}}},"cloud":{"provider":"","instance":{},"machine":{},"account":{},"project":{},"service":{}}}}`, extension.Version) - desiredOutputMetrics := fmt.Sprintf(`{"metricset":{"samples":{"aws.lambda.metrics.BilledDuration":{"value":183},"aws.lambda.metrics.ColdStartDuration":{"value":422.9700012207031},"aws.lambda.metrics.Timeout":{"value":5000},"system.memory.total":{"value":1.34217728e+08},"system.memory.actual.free":{"value":5.4525952e+07},"aws.lambda.metrics.Duration":{"value":182.42999267578125}},"timestamp":%d,"tags":{"faas.execution":"6f7f0961f83442118a7af6fe80b88d56","faas.id":"arn:aws:lambda:us-east-2:123456789012:function:custom-runtime","faas.coldstart":"true"}}}`, timestamp.UnixNano()/1e3) + desiredOutputMetrics := fmt.Sprintf(`{"metricset":{"samples":{"aws.lambda.metrics.billed_duration":{"value":183},"aws.lambda.metrics.cold_start_duration":{"value":422.9700012207031},"aws.lambda.metrics.timeout":{"value":5000},"system.memory.total":{"value":1.34217728e+08},"system.memory.actual.free":{"value":5.4525952e+07},"aws.lambda.metrics.duration":{"value":182.42999267578125}},"timestamp":%d,"tags":{"faas.execution":"6f7f0961f83442118a7af6fe80b88d56","faas.id":"arn:aws:lambda:us-east-2:123456789012:function:custom-runtime","faas.coldstart":"true"}}}`, timestamp.UnixNano()/1e3) rawBytes, err := ProcessPlatformReport(context.Background(), &mc, &event, logEvent) require.NoError(t, err) diff --git a/apm-lambda-extension/main_test.go b/apm-lambda-extension/main_test.go index 553baff6..a1bbdb89 100644 --- a/apm-lambda-extension/main_test.go +++ b/apm-lambda-extension/main_test.go @@ -596,9 +596,9 @@ func TestMetricsWithoutMetadata(t *testing.T) { eventQueueGenerator(eventsChain, eventsChannel) assert.NotPanics(t, main) - assert.Contains(t, apmServerInternals.Data, `aws.lambda.metrics.BilledDuration":{"value":60`) - assert.Contains(t, apmServerInternals.Data, `aws.lambda.metrics.Duration":{"value":59.9`) - assert.Contains(t, apmServerInternals.Data, `aws.lambda.metrics.ColdStartDuration":{"value":500`) + assert.Contains(t, apmServerInternals.Data, `aws.lambda.metrics.billed_duration":{"value":60`) + assert.Contains(t, apmServerInternals.Data, `aws.lambda.metrics.duration":{"value":59.9`) + assert.Contains(t, apmServerInternals.Data, `aws.lambda.metrics.cold_start_duration":{"value":500`) } // TestMetricsWithMetadata checks if the extension sends metrics corresponding to invocation n during invocation @@ -617,7 +617,7 @@ func TestMetricsWithMetadata(t *testing.T) { assert.NotPanics(t, main) assert.Contains(t, apmServerInternals.Data, `{"metadata":{"service":{"name":"1234_service-12a3","version":"5.1.3","environment":"staging","agent":{"name":"elastic-node","version":"3.14.0"},"framework":{"name":"Express","version":"1.2.3"},"language":{"name":"ecmascript","version":"8"},"runtime":{"name":"node","version":"8.0.0"},"node":{"configured_name":"node-123"}},"user":{"username":"bar","id":"123user","email":"bar@user.com"},"labels":{"tag0":null,"tag1":"one","tag2":2},"process":{"pid":1234,"ppid":6789,"title":"node","argv":["node","server.js"]},"system":{"architecture":"x64","hostname":"prod1.example.com","platform":"darwin","container":{"id":"container-id"},"kubernetes":{"namespace":"namespace1","node":{"name":"node-name"},"pod":{"name":"pod-name","uid":"pod-uid"}}},"cloud":{"provider":"cloud_provider","region":"cloud_region","availability_zone":"cloud_availability_zone","instance":{"id":"instance_id","name":"instance_name"},"machine":{"type":"machine_type"},"account":{"id":"account_id","name":"account_name"},"project":{"id":"project_id","name":"project_name"},"service":{"name":"lambda"}}}}`) - assert.Contains(t, apmServerInternals.Data, `aws.lambda.metrics.BilledDuration":{"value":60`) - assert.Contains(t, apmServerInternals.Data, `aws.lambda.metrics.Duration":{"value":59.9`) - assert.Contains(t, apmServerInternals.Data, `aws.lambda.metrics.ColdStartDuration":{"value":500`) + assert.Contains(t, apmServerInternals.Data, `aws.lambda.metrics.billed_duration":{"value":60`) + assert.Contains(t, apmServerInternals.Data, `aws.lambda.metrics.duration":{"value":59.9`) + assert.Contains(t, apmServerInternals.Data, `aws.lambda.metrics.cold_start_duration":{"value":500`) } From 62501a6f9b96022725d689251fe4f843ce2cb7df Mon Sep 17 00:00:00 2001 From: Jean-Louis Voiseux <48380853+jlvoiseux@users.noreply.github.com> Date: Fri, 3 Jun 2022 17:48:54 +0200 Subject: [PATCH 16/21] Update process_metadata.go --- apm-lambda-extension/extension/process_metadata.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/apm-lambda-extension/extension/process_metadata.go b/apm-lambda-extension/extension/process_metadata.go index b23eee46..8ba5dca6 100644 --- a/apm-lambda-extension/extension/process_metadata.go +++ b/apm-lambda-extension/extension/process_metadata.go @@ -41,10 +41,9 @@ func ProcessMetadata(data AgentData) ([]byte, error) { return nil, errors.New(fmt.Sprintf("Error uncompressing agent data for metadata extraction : %v", err)) } scanner := bufio.NewScanner(strings.NewReader(string(uncompressedData))) - for scanner.Scan() { - if strings.Contains(strings.ToLower(scanner.Text()), "metadata") { - return scanner.Bytes(), nil - } + scanner.Scan() + if strings.Contains(strings.ToLower(scanner.Text()), "metadata") { + return scanner.Bytes(), nil } return nil, errors.New("No metadata found in APM agent payload") } From 3e9956f78d4ff0c0815132b88927711077aa5020 Mon Sep 17 00:00:00 2001 From: Jean-Louis Voiseux <48380853+jlvoiseux@users.noreply.github.com> Date: Fri, 3 Jun 2022 17:59:37 +0200 Subject: [PATCH 17/21] Small fixes (go mod, metrics names) --- apm-lambda-extension/go.mod | 4 ---- apm-lambda-extension/go.sum | 8 -------- apm-lambda-extension/logsapi/process_metrics.go | 6 +++--- 3 files changed, 3 insertions(+), 15 deletions(-) diff --git a/apm-lambda-extension/go.mod b/apm-lambda-extension/go.mod index 452fb283..384e887d 100644 --- a/apm-lambda-extension/go.mod +++ b/apm-lambda-extension/go.mod @@ -16,7 +16,6 @@ require ( ) require ( - github.com/tidwall/sjson v1.2.4 go.elastic.co/apm v1.15.0 go.elastic.co/fastjson v1.1.0 ) @@ -26,8 +25,5 @@ require ( github.com/google/go-cmp v0.5.6 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/santhosh-tekuri/jsonschema v1.2.4 // indirect - github.com/tidwall/gjson v1.12.1 // indirect - github.com/tidwall/match v1.1.1 // indirect - github.com/tidwall/pretty v1.2.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) diff --git a/apm-lambda-extension/go.sum b/apm-lambda-extension/go.sum index fd32a941..b35835db 100644 --- a/apm-lambda-extension/go.sum +++ b/apm-lambda-extension/go.sum @@ -39,14 +39,6 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/tidwall/gjson v1.12.1 h1:ikuZsLdhr8Ws0IdROXUS1Gi4v9Z4pGqpX/CvJkxvfpo= -github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= -github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= -github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= -github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/tidwall/sjson v1.2.4 h1:cuiLzLnaMeBhRmEv00Lpk3tkYrcxpmbU81tAY4Dw0tc= -github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp19pSNM= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.elastic.co/apm v1.15.0 h1:uPk2g/whK7c7XiZyz/YCUnAUBNPiyNeE3ARX3G6Gx7Q= diff --git a/apm-lambda-extension/logsapi/process_metrics.go b/apm-lambda-extension/logsapi/process_metrics.go index e0626af8..eddcba3b 100644 --- a/apm-lambda-extension/logsapi/process_metrics.go +++ b/apm-lambda-extension/logsapi/process_metrics.go @@ -87,9 +87,9 @@ func ProcessPlatformReport(ctx context.Context, metadataContainer *extension.Met metricsContainer.Add("system.memory.actual.free", float64(platformReportMetrics.MemorySizeMB-platformReportMetrics.MaxMemoryUsedMB)*convMB2Bytes) // Unit : Bytes // Raw Metrics - metricsContainer.Add("aws.lambda.metrics.duration", float64(platformReportMetrics.DurationMs)) // Unit : Milliseconds - metricsContainer.Add("aws.lambda.metrics.billed_duration", float64(platformReportMetrics.BilledDurationMs)) // Unit : Milliseconds - metricsContainer.Add("aws.lambda.metrics.cold_start_duration", float64(platformReportMetrics.InitDurationMs)) // Unit : Milliseconds + metricsContainer.Add("aws.lambda.metrics.duration", float64(platformReportMetrics.DurationMs)) // Unit : Milliseconds + metricsContainer.Add("aws.lambda.metrics.billed_duration", float64(platformReportMetrics.BilledDurationMs)) // Unit : Milliseconds + metricsContainer.Add("aws.lambda.metrics.coldstart_duration", float64(platformReportMetrics.InitDurationMs)) // Unit : Milliseconds // In AWS Lambda, the Timeout is configured as an integer number of seconds. We use this assumption to derive the Timeout from // - The epoch corresponding to the end of the current invocation (its "deadline") // - The epoch corresponding to the start of the current invocation From daf452324a1045e03aed551e672a5155bdaafde4 Mon Sep 17 00:00:00 2001 From: Jean-Louis Voiseux <48380853+jlvoiseux@users.noreply.github.com> Date: Fri, 3 Jun 2022 18:36:08 +0200 Subject: [PATCH 18/21] Update notice --- apm-lambda-extension/NOTICE.txt | 280 +++++++++++++++++++++ apm-lambda-extension/dependencies.asciidoc | 3 + 2 files changed, 283 insertions(+) diff --git a/apm-lambda-extension/NOTICE.txt b/apm-lambda-extension/NOTICE.txt index 99c203f9..ad0fb9a5 100644 --- a/apm-lambda-extension/NOTICE.txt +++ b/apm-lambda-extension/NOTICE.txt @@ -151,6 +151,217 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +-------------------------------------------------------------------------------- +Module : go.elastic.co/apm +Version : v1.15.0 +Time : 2021-12-08T03:18:47Z +Licence : Apache-2.0 + +Contents of probable licence file $GOMODCACHE/go.elastic.co/apm@v1.15.0/LICENSE: + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2018 Elasticsearch BV + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + -------------------------------------------------------------------------------- Module : go.elastic.co/ecszap Version : v1.0.1 @@ -362,6 +573,39 @@ Contents of probable licence file $GOMODCACHE/go.elastic.co/ecszap@v1.0.1/LICENS See the License for the specific language governing permissions and limitations under the License. +-------------------------------------------------------------------------------- +Module : go.elastic.co/fastjson +Version : v1.1.0 +Time : 2020-05-11T07:15:19Z +Licence : MIT + +Contents of probable licence file $GOMODCACHE/go.elastic.co/fastjson@v1.1.0/LICENSE: + +Copyright 2018 Elasticsearch BV + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +--- + +Copyright (c) 2016 Mail.Ru Group + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + -------------------------------------------------------------------------------- Module : go.uber.org/zap Version : v1.21.0 @@ -821,6 +1065,42 @@ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +-------------------------------------------------------------------------------- +Module : github.com/santhosh-tekuri/jsonschema +Version : v1.2.4 +Time : 2018-12-06T11:30:25Z +Licence : BSD-3-Clause + +Contents of probable licence file $GOMODCACHE/github.com/santhosh-tekuri/jsonschema@v1.2.4/LICENSE: + +Copyright (c) 2017 Santhosh Kumar Tekuri. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + -------------------------------------------------------------------------------- Module : go.uber.org/atomic Version : v1.9.0 diff --git a/apm-lambda-extension/dependencies.asciidoc b/apm-lambda-extension/dependencies.asciidoc index fe39d0a8..4a67b3cc 100644 --- a/apm-lambda-extension/dependencies.asciidoc +++ b/apm-lambda-extension/dependencies.asciidoc @@ -18,7 +18,9 @@ This page lists the third-party dependencies used to build {n}. | link:https://github.com/joho/godotenv[$$github.com/joho/godotenv$$] | v1.4.0 | MIT | link:https://github.com/pkg/errors[$$github.com/pkg/errors$$] | v0.9.1 | BSD-2-Clause | link:https://github.com/stretchr/testify[$$github.com/stretchr/testify$$] | v1.7.0 | MIT +| link:https://go.elastic.co/apm[$$go.elastic.co/apm$$] | v1.15.0 | Apache-2.0 | link:https://go.elastic.co/ecszap[$$go.elastic.co/ecszap$$] | v1.0.1 | Apache-2.0 +| link:https://go.elastic.co/fastjson[$$go.elastic.co/fastjson$$] | v1.1.0 | MIT | link:https://go.uber.org/zap[$$go.uber.org/zap$$] | v1.21.0 | MIT | link:https://gotest.tools[$$gotest.tools$$] | v2.2.0+incompatible | Apache-2.0 |=== @@ -39,6 +41,7 @@ This page lists the third-party dependencies used to build {n}. | link:https://github.com/kr/text[$$github.com/kr/text$$] | v0.1.0 | MIT | link:https://github.com/magefile/mage[$$github.com/magefile/mage$$] | v1.13.0 | Apache-2.0 | link:https://github.com/pmezard/go-difflib[$$github.com/pmezard/go-difflib$$] | v1.0.0 | BSD-3-Clause +| link:https://github.com/santhosh-tekuri/jsonschema[$$github.com/santhosh-tekuri/jsonschema$$] | v1.2.4 | BSD-3-Clause | link:https://go.uber.org/atomic[$$go.uber.org/atomic$$] | v1.9.0 | MIT | link:https://go.uber.org/goleak[$$go.uber.org/goleak$$] | v1.1.11 | MIT | link:https://go.uber.org/multierr[$$go.uber.org/multierr$$] | v1.8.0 | MIT From 9df299ffe275dff7c995baf9b55e320a25036fb7 Mon Sep 17 00:00:00 2001 From: Jean-Louis Voiseux <48380853+jlvoiseux@users.noreply.github.com> Date: Fri, 3 Jun 2022 20:00:42 +0200 Subject: [PATCH 19/21] Fix typo --- apm-lambda-extension/logsapi/process_metrics_test.go | 2 +- apm-lambda-extension/main_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apm-lambda-extension/logsapi/process_metrics_test.go b/apm-lambda-extension/logsapi/process_metrics_test.go index bb683c90..89d6649f 100644 --- a/apm-lambda-extension/logsapi/process_metrics_test.go +++ b/apm-lambda-extension/logsapi/process_metrics_test.go @@ -74,7 +74,7 @@ func Test_processPlatformReport(t *testing.T) { desiredOutputMetadata := fmt.Sprintf(`{"metadata":{"service":{"agent":{"name":"apm-lambda-extension","version":"%s"},"framework":{"name":"AWS Lambda","version":""},"language":{"name":"python","version":"3.9.8"},"runtime":{"name":"","version":""},"node":{}},"user":{},"process":{"pid":0},"system":{"container":{"id":""},"kubernetes":{"node":{},"pod":{}}},"cloud":{"provider":"","instance":{},"machine":{},"account":{},"project":{},"service":{}}}}`, extension.Version) - desiredOutputMetrics := fmt.Sprintf(`{"metricset":{"samples":{"aws.lambda.metrics.billed_duration":{"value":183},"aws.lambda.metrics.cold_start_duration":{"value":422.9700012207031},"aws.lambda.metrics.timeout":{"value":5000},"system.memory.total":{"value":1.34217728e+08},"system.memory.actual.free":{"value":5.4525952e+07},"aws.lambda.metrics.duration":{"value":182.42999267578125}},"timestamp":%d,"tags":{"faas.execution":"6f7f0961f83442118a7af6fe80b88d56","faas.id":"arn:aws:lambda:us-east-2:123456789012:function:custom-runtime","faas.coldstart":"true"}}}`, timestamp.UnixNano()/1e3) + desiredOutputMetrics := fmt.Sprintf(`{"metricset":{"samples":{"aws.lambda.metrics.billed_duration":{"value":183},"aws.lambda.metrics.coldstart_duration":{"value":422.9700012207031},"aws.lambda.metrics.timeout":{"value":5000},"system.memory.total":{"value":1.34217728e+08},"system.memory.actual.free":{"value":5.4525952e+07},"aws.lambda.metrics.duration":{"value":182.42999267578125}},"timestamp":%d,"tags":{"faas.execution":"6f7f0961f83442118a7af6fe80b88d56","faas.id":"arn:aws:lambda:us-east-2:123456789012:function:custom-runtime","faas.coldstart":"true"}}}`, timestamp.UnixNano()/1e3) rawBytes, err := ProcessPlatformReport(context.Background(), &mc, &event, logEvent) require.NoError(t, err) diff --git a/apm-lambda-extension/main_test.go b/apm-lambda-extension/main_test.go index a1bbdb89..9c56ed6b 100644 --- a/apm-lambda-extension/main_test.go +++ b/apm-lambda-extension/main_test.go @@ -598,7 +598,7 @@ func TestMetricsWithoutMetadata(t *testing.T) { assert.Contains(t, apmServerInternals.Data, `aws.lambda.metrics.billed_duration":{"value":60`) assert.Contains(t, apmServerInternals.Data, `aws.lambda.metrics.duration":{"value":59.9`) - assert.Contains(t, apmServerInternals.Data, `aws.lambda.metrics.cold_start_duration":{"value":500`) + assert.Contains(t, apmServerInternals.Data, `aws.lambda.metrics.coldstart_duration":{"value":500`) } // TestMetricsWithMetadata checks if the extension sends metrics corresponding to invocation n during invocation @@ -619,5 +619,5 @@ func TestMetricsWithMetadata(t *testing.T) { assert.Contains(t, apmServerInternals.Data, `{"metadata":{"service":{"name":"1234_service-12a3","version":"5.1.3","environment":"staging","agent":{"name":"elastic-node","version":"3.14.0"},"framework":{"name":"Express","version":"1.2.3"},"language":{"name":"ecmascript","version":"8"},"runtime":{"name":"node","version":"8.0.0"},"node":{"configured_name":"node-123"}},"user":{"username":"bar","id":"123user","email":"bar@user.com"},"labels":{"tag0":null,"tag1":"one","tag2":2},"process":{"pid":1234,"ppid":6789,"title":"node","argv":["node","server.js"]},"system":{"architecture":"x64","hostname":"prod1.example.com","platform":"darwin","container":{"id":"container-id"},"kubernetes":{"namespace":"namespace1","node":{"name":"node-name"},"pod":{"name":"pod-name","uid":"pod-uid"}}},"cloud":{"provider":"cloud_provider","region":"cloud_region","availability_zone":"cloud_availability_zone","instance":{"id":"instance_id","name":"instance_name"},"machine":{"type":"machine_type"},"account":{"id":"account_id","name":"account_name"},"project":{"id":"project_id","name":"project_name"},"service":{"name":"lambda"}}}}`) assert.Contains(t, apmServerInternals.Data, `aws.lambda.metrics.billed_duration":{"value":60`) assert.Contains(t, apmServerInternals.Data, `aws.lambda.metrics.duration":{"value":59.9`) - assert.Contains(t, apmServerInternals.Data, `aws.lambda.metrics.cold_start_duration":{"value":500`) + assert.Contains(t, apmServerInternals.Data, `aws.lambda.metrics.coldstart_duration":{"value":500`) } From b220be934d2727b79c4893b89febcda6c0b85094 Mon Sep 17 00:00:00 2001 From: Jean-Louis Voiseux <48380853+jlvoiseux@users.noreply.github.com> Date: Tue, 7 Jun 2022 17:37:24 +0200 Subject: [PATCH 20/21] Fix : Use v2 APM model --- apm-lambda-extension/NOTICE.txt | 8 ++--- apm-lambda-extension/dependencies.asciidoc | 2 +- apm-lambda-extension/go.mod | 2 +- apm-lambda-extension/go.sum | 34 +++++++++++++++---- .../logsapi/process_metrics.go | 2 +- 5 files changed, 34 insertions(+), 14 deletions(-) diff --git a/apm-lambda-extension/NOTICE.txt b/apm-lambda-extension/NOTICE.txt index ad0fb9a5..64b63cfe 100644 --- a/apm-lambda-extension/NOTICE.txt +++ b/apm-lambda-extension/NOTICE.txt @@ -152,12 +152,12 @@ SOFTWARE. -------------------------------------------------------------------------------- -Module : go.elastic.co/apm -Version : v1.15.0 -Time : 2021-12-08T03:18:47Z +Module : go.elastic.co/apm/v2 +Version : v2.1.0 +Time : 2022-05-20T09:45:34Z Licence : Apache-2.0 -Contents of probable licence file $GOMODCACHE/go.elastic.co/apm@v1.15.0/LICENSE: +Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/v2@v2.1.0/LICENSE: Apache License Version 2.0, January 2004 diff --git a/apm-lambda-extension/dependencies.asciidoc b/apm-lambda-extension/dependencies.asciidoc index 4a67b3cc..91e33c66 100644 --- a/apm-lambda-extension/dependencies.asciidoc +++ b/apm-lambda-extension/dependencies.asciidoc @@ -18,7 +18,7 @@ This page lists the third-party dependencies used to build {n}. | link:https://github.com/joho/godotenv[$$github.com/joho/godotenv$$] | v1.4.0 | MIT | link:https://github.com/pkg/errors[$$github.com/pkg/errors$$] | v0.9.1 | BSD-2-Clause | link:https://github.com/stretchr/testify[$$github.com/stretchr/testify$$] | v1.7.0 | MIT -| link:https://go.elastic.co/apm[$$go.elastic.co/apm$$] | v1.15.0 | Apache-2.0 +| link:https://go.elastic.co/apm/v2[$$go.elastic.co/apm/v2$$] | v2.1.0 | Apache-2.0 | link:https://go.elastic.co/ecszap[$$go.elastic.co/ecszap$$] | v1.0.1 | Apache-2.0 | link:https://go.elastic.co/fastjson[$$go.elastic.co/fastjson$$] | v1.1.0 | MIT | link:https://go.uber.org/zap[$$go.uber.org/zap$$] | v1.21.0 | MIT diff --git a/apm-lambda-extension/go.mod b/apm-lambda-extension/go.mod index 384e887d..eb902e48 100644 --- a/apm-lambda-extension/go.mod +++ b/apm-lambda-extension/go.mod @@ -16,7 +16,7 @@ require ( ) require ( - go.elastic.co/apm v1.15.0 + go.elastic.co/apm/v2 v2.1.0 go.elastic.co/fastjson v1.1.0 ) diff --git a/apm-lambda-extension/go.sum b/apm-lambda-extension/go.sum index b35835db..9ff7a63a 100644 --- a/apm-lambda-extension/go.sum +++ b/apm-lambda-extension/go.sum @@ -4,10 +4,11 @@ github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZx github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/elastic/go-licenser v0.3.1/go.mod h1:D8eNQk70FOCVBl3smCGQt/lv7meBeQno2eI1S5apiHQ= -github.com/elastic/go-sysinfo v1.1.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= +github.com/elastic/go-licenser v0.4.0/go.mod h1:V56wHMpmdURfibNBggaSBfqgPxyT1Tldns1i87iTEvU= +github.com/elastic/go-sysinfo v1.7.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/elastic/go-windows v1.0.1/go.mod h1:FoVvqWSun28vaDQPbj2Elfc0JahhPB7WQEGa3c814Ss= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= @@ -32,6 +33,7 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/procfs v0.0.0-20190425082905-87a4384529e0/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/santhosh-tekuri/jsonschema v1.2.4 h1:hNhW8e7t+H1vgY+1QeEQpveR6D4+OwKPXCfD2aieJis= github.com/santhosh-tekuri/jsonschema v1.2.4/go.mod h1:TEAUOeZSmIxTTuHatJzrvARHiuO9LYd+cIxzgEHCQI4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -41,8 +43,10 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.elastic.co/apm v1.15.0 h1:uPk2g/whK7c7XiZyz/YCUnAUBNPiyNeE3ARX3G6Gx7Q= -go.elastic.co/apm v1.15.0/go.mod h1:dylGv2HKR0tiCV+wliJz1KHtDyuD8SPe69oV7VyK6WY= +github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.elastic.co/apm/v2 v2.1.0 h1:rkJSHE4ggekHhUR5v0KKkoMbrRSJN8YoBiEgQnkV1OY= +go.elastic.co/apm/v2 v2.1.0/go.mod h1:KGQn56LtRmkQjt2qw4+c1Jz8gv9rCBUU/m21uxrqcps= go.elastic.co/ecszap v1.0.1 h1:mBxqEJAEXBlpi5+scXdzL7LTFGogbuxipJC0KTZicyA= go.elastic.co/ecszap v1.0.1/go.mod h1:SVjazT+QgNeHSGOCUHvRgN+ZRj5FkB7IXQQsncdF57A= go.elastic.co/fastjson v1.1.0 h1:3MrGBWWVIxe/xvsbpghtkFoPciPhOCmjsR/HfwEeQR4= @@ -60,36 +64,50 @@ go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191025021431-6c3a3bfe00ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211102192858-4dd72447c267/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200509030707-2212a7e161a5/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= +golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -98,6 +116,7 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -107,3 +126,4 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= +howett.net/plist v1.0.0/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g= diff --git a/apm-lambda-extension/logsapi/process_metrics.go b/apm-lambda-extension/logsapi/process_metrics.go index eddcba3b..f857c85d 100644 --- a/apm-lambda-extension/logsapi/process_metrics.go +++ b/apm-lambda-extension/logsapi/process_metrics.go @@ -24,7 +24,7 @@ import ( "elastic/apm-lambda-extension/extension" - "go.elastic.co/apm/model" + "go.elastic.co/apm/v2/model" "go.elastic.co/fastjson" ) From 16f5468d04c71c19a50d4c09ca50df70e62001d2 Mon Sep 17 00:00:00 2001 From: Jean-Louis Voiseux <48380853+jlvoiseux@users.noreply.github.com> Date: Wed, 8 Jun 2022 10:07:16 +0200 Subject: [PATCH 21/21] Resolve conflicts with main --- apm-lambda-extension/NOTICE.txt | 244 +++++++++++++++++++++ apm-lambda-extension/dependencies.asciidoc | 4 + apm-lambda-extension/go.sum | 50 +++++ 3 files changed, 298 insertions(+) diff --git a/apm-lambda-extension/NOTICE.txt b/apm-lambda-extension/NOTICE.txt index 9bf9a42c..79ec13bf 100644 --- a/apm-lambda-extension/NOTICE.txt +++ b/apm-lambda-extension/NOTICE.txt @@ -285,6 +285,217 @@ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +-------------------------------------------------------------------------------- +Module : go.elastic.co/apm/v2 +Version : v2.1.0 +Time : 2022-05-20T09:45:34Z +Licence : Apache-2.0 + +Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/v2@v2.1.0/LICENSE: + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2018 Elasticsearch BV + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + -------------------------------------------------------------------------------- Module : go.elastic.co/ecszap Version : v1.0.1 @@ -496,6 +707,39 @@ Contents of probable licence file $GOMODCACHE/go.elastic.co/ecszap@v1.0.1/LICENS See the License for the specific language governing permissions and limitations under the License. +-------------------------------------------------------------------------------- +Module : go.elastic.co/fastjson +Version : v1.1.0 +Time : 2020-05-11T07:15:19Z +Licence : MIT + +Contents of probable licence file $GOMODCACHE/go.elastic.co/fastjson@v1.1.0/LICENSE: + +Copyright 2018 Elasticsearch BV + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +--- + +Copyright (c) 2016 Mail.Ru Group + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + -------------------------------------------------------------------------------- Module : go.uber.org/atomic Version : v1.9.0 diff --git a/apm-lambda-extension/dependencies.asciidoc b/apm-lambda-extension/dependencies.asciidoc index 343e9349..a7629211 100644 --- a/apm-lambda-extension/dependencies.asciidoc +++ b/apm-lambda-extension/dependencies.asciidoc @@ -17,8 +17,12 @@ This page lists the third-party dependencies used to build {n}. | link:https://github.com/aws/aws-sdk-go[$$github.com/aws/aws-sdk-go$$] | v1.44.27 | Apache-2.0 | link:https://github.com/jmespath/go-jmespath[$$github.com/jmespath/go-jmespath$$] | v0.4.0 | Apache-2.0 | link:https://github.com/pkg/errors[$$github.com/pkg/errors$$] | v0.9.1 | BSD-2-Clause +| link:https://go.elastic.co/apm/v2[$$go.elastic.co/apm/v2$$] | v2.1.0 | Apache-2.0 | link:https://go.elastic.co/ecszap[$$go.elastic.co/ecszap$$] | v1.0.1 | Apache-2.0 +| link:https://go.elastic.co/fastjson[$$go.elastic.co/fastjson$$] | v1.1.0 | MIT | link:https://go.uber.org/atomic[$$go.uber.org/atomic$$] | v1.9.0 | MIT | link:https://go.uber.org/multierr[$$go.uber.org/multierr$$] | v1.8.0 | MIT | link:https://go.uber.org/zap[$$go.uber.org/zap$$] | v1.21.0 | MIT |=== + + diff --git a/apm-lambda-extension/go.sum b/apm-lambda-extension/go.sum index 4a908e66..71cb8653 100644 --- a/apm-lambda-extension/go.sum +++ b/apm-lambda-extension/go.sum @@ -1,3 +1,4 @@ +github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aws/aws-sdk-go v1.44.27 h1:8CMspeZSrewnbvAwgl8qo5R7orDLwQnTGBf/OKPiHxI= github.com/aws/aws-sdk-go v1.44.27/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= @@ -5,14 +6,22 @@ github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZx github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/elastic/go-licenser v0.4.0/go.mod h1:V56wHMpmdURfibNBggaSBfqgPxyT1Tldns1i87iTEvU= +github.com/elastic/go-sysinfo v1.7.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= +github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU= +github.com/elastic/go-windows v1.0.1/go.mod h1:FoVvqWSun28vaDQPbj2Elfc0JahhPB7WQEGa3c814Ss= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jcchavezs/porto v0.1.0/go.mod h1:fESH0gzDHiutHRdX2hv27ojnOVFco37hg1W6E9EZF4A= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg= github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= @@ -23,18 +32,31 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/magefile/mage v1.9.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magefile/mage v1.13.0 h1:XtLJl8bcCM7EFoO8FyH8XK3t7G5hQAeK+i4tq+veT9M= github.com/magefile/mage v1.13.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/procfs v0.0.0-20190425082905-87a4384529e0/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/santhosh-tekuri/jsonschema v1.2.4 h1:hNhW8e7t+H1vgY+1QeEQpveR6D4+OwKPXCfD2aieJis= +github.com/santhosh-tekuri/jsonschema v1.2.4/go.mod h1:TEAUOeZSmIxTTuHatJzrvARHiuO9LYd+cIxzgEHCQI4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.elastic.co/apm/v2 v2.1.0 h1:rkJSHE4ggekHhUR5v0KKkoMbrRSJN8YoBiEgQnkV1OY= +go.elastic.co/apm/v2 v2.1.0/go.mod h1:KGQn56LtRmkQjt2qw4+c1Jz8gv9rCBUU/m21uxrqcps= go.elastic.co/ecszap v1.0.1 h1:mBxqEJAEXBlpi5+scXdzL7LTFGogbuxipJC0KTZicyA= go.elastic.co/ecszap v1.0.1/go.mod h1:SVjazT+QgNeHSGOCUHvRgN+ZRj5FkB7IXQQsncdF57A= +go.elastic.co/fastjson v1.1.0 h1:3MrGBWWVIxe/xvsbpghtkFoPciPhOCmjsR/HfwEeQR4= +go.elastic.co/fastjson v1.1.0/go.mod h1:boNGISWMjQsUPy/t6yqt2/1Wx4YNPSe+mZjlyw9vKKI= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -48,30 +70,54 @@ go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191025021431-6c3a3bfe00ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211102192858-4dd72447c267/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200509030707-2212a7e161a5/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= +golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -80,6 +126,8 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -87,3 +135,5 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= +howett.net/plist v1.0.0/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g=