From 1a2bd1c451b3b10ff0471757b218605e3c13dd5d Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 17 Jun 2020 10:23:30 +0200 Subject: [PATCH 001/175] Normalize DD_TAGS to comma-separated --- extra/datadog.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index 7e48212..f854e6d 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -46,10 +46,10 @@ done DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} BUILDPACKVERSION="dev" -DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" +DYNO_TAGS="dyno:$DYNO,dynotype:$DYNOTYPE,buildpackversion:$BUILDPACKVERSION" if [ -n "$HEROKU_APP_NAME" ]; then - DYNO_TAGS="$DYNO_TAGS appname:$HEROKU_APP_NAME" + DYNO_TAGS="$DYNO_TAGS,appname:$HEROKU_APP_NAME" fi # Uncomment APM configs and add the log file location. @@ -154,8 +154,8 @@ fi # Convert comma delimited tags from env vars to yaml if [ -n "$DD_TAGS" ]; then - DD_TAGS_NORMALIZED="$(sed "s/,[ ]\?/\ /g" <<< "$DD_TAGS")" - DD_TAGS="$DYNO_TAGS $DD_TAGS_NORMALIZED" + DD_TAGS_NORMALIZED="$(sed "s/,\?[ ]\+/,/g" <<< "$DD_TAGS")" + DD_TAGS="$DYNO_TAGS,$DD_TAGS_NORMALIZED" else DD_TAGS="$DYNO_TAGS" fi @@ -165,7 +165,7 @@ if [ "$DD_LOG_LEVEL_LOWER" == "debug" ]; then echo "[DEBUG] Buildpack normalized tags: $DD_TAGS_NORMALIZED" fi -DD_TAGS_YAML="tags:\n - $(sed "s/\ /\\\n - /g" <<< "$DD_TAGS")" +DD_TAGS_YAML="tags:\n - $(sed "s/,/\\\n - /g" <<< "$DD_TAGS")" # Inject tags after example tags. # Config files for agent versions 6.11 and earlier: From b3c13ec1668eb0f1a502c12c45b04aa1597095d4 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 17 Jun 2020 11:23:52 +0200 Subject: [PATCH 002/175] Fix tests --- test/compile_and_run_test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/compile_and_run_test.sh b/test/compile_and_run_test.sh index c6ddfbb..4e6cb8d 100644 --- a/test/compile_and_run_test.sh +++ b/test/compile_and_run_test.sh @@ -29,7 +29,7 @@ compileAndRunVersion() assertCaptured "Installing Datadog runner" export HOME=${BUILD_DIR} - export DD_TAGS="sampletag1:sample,sametag2:sample, sampletag3 sampletag4" + export DD_TAGS="sampletag1:sample,sametag2:sample, sampletag3 sampletag4" chmod +x ${BUILDPACK_HOME}/extra/datadog.sh capture ${BUILDPACK_HOME}/extra/datadog.sh assertFileNotContains "error while loading shared libraries" ${STD_ERR} @@ -37,7 +37,7 @@ compileAndRunVersion() assertCaptured "Starting Datadog Agent" assertCaptured "Starting Datadog Trace Agent" assertNotCaptured "The Datadog Agent has been disabled" - assertCaptured "[DEBUG] Buildpack normalized tags: sampletag1:sample sametag2:sample sampletag3 sampletag4" + assertCaptured "[DEBUG] Buildpack normalized tags: sampletag1:sample,sametag2:sample,sampletag3,sampletag4" } From b4bb2f9438aff92e3929d1fb9333f420632a3f9e Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 17 Jun 2020 12:25:03 +0200 Subject: [PATCH 003/175] Move to 6.20.1 and 7.20.1 for agent pinned version --- bin/compile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/compile b/bin/compile index d5474ad..523dbf8 100644 --- a/bin/compile +++ b/bin/compile @@ -11,8 +11,8 @@ set -o pipefail # set -x # Set agent pinned version -DD_AGENT_PINNED_VERSION_6="6.19.0-1" -DD_AGENT_PINNED_VERSION_7="7.19.0-1" +DD_AGENT_PINNED_VERSION_6="6.20.1-1" +DD_AGENT_PINNED_VERSION_7="7.20.1-1" # Parse and derive params BUILD_DIR=$1 From ae61001a72d53315a95062d9f9a8ee9a8d68dae3 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 17 Jun 2020 17:40:46 +0200 Subject: [PATCH 004/175] Pin agent versions .20.2 --- bin/compile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/compile b/bin/compile index 523dbf8..5ace375 100644 --- a/bin/compile +++ b/bin/compile @@ -11,8 +11,8 @@ set -o pipefail # set -x # Set agent pinned version -DD_AGENT_PINNED_VERSION_6="6.20.1-1" -DD_AGENT_PINNED_VERSION_7="7.20.1-1" +DD_AGENT_PINNED_VERSION_6="6.20.2-1" +DD_AGENT_PINNED_VERSION_7="7.20.2-1" # Parse and derive params BUILD_DIR=$1 From 3969bfeca15e11fb4f2d670b6c186497c3954c88 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 17 Jun 2020 18:07:26 +0200 Subject: [PATCH 005/175] Revert "Merge pull request #189 from DataDog/ara.pulido/workaround_ddtags" This reverts commit 80805c1be8d3a61841180c65e45e65d98ea77c4b, reversing changes made to f98c0e46393ee5a9ea9d8b760d8e33b91023a8e6. --- extra/datadog.sh | 10 +++++----- test/compile_and_run_test.sh | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index f854e6d..7e48212 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -46,10 +46,10 @@ done DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} BUILDPACKVERSION="dev" -DYNO_TAGS="dyno:$DYNO,dynotype:$DYNOTYPE,buildpackversion:$BUILDPACKVERSION" +DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" if [ -n "$HEROKU_APP_NAME" ]; then - DYNO_TAGS="$DYNO_TAGS,appname:$HEROKU_APP_NAME" + DYNO_TAGS="$DYNO_TAGS appname:$HEROKU_APP_NAME" fi # Uncomment APM configs and add the log file location. @@ -154,8 +154,8 @@ fi # Convert comma delimited tags from env vars to yaml if [ -n "$DD_TAGS" ]; then - DD_TAGS_NORMALIZED="$(sed "s/,\?[ ]\+/,/g" <<< "$DD_TAGS")" - DD_TAGS="$DYNO_TAGS,$DD_TAGS_NORMALIZED" + DD_TAGS_NORMALIZED="$(sed "s/,[ ]\?/\ /g" <<< "$DD_TAGS")" + DD_TAGS="$DYNO_TAGS $DD_TAGS_NORMALIZED" else DD_TAGS="$DYNO_TAGS" fi @@ -165,7 +165,7 @@ if [ "$DD_LOG_LEVEL_LOWER" == "debug" ]; then echo "[DEBUG] Buildpack normalized tags: $DD_TAGS_NORMALIZED" fi -DD_TAGS_YAML="tags:\n - $(sed "s/,/\\\n - /g" <<< "$DD_TAGS")" +DD_TAGS_YAML="tags:\n - $(sed "s/\ /\\\n - /g" <<< "$DD_TAGS")" # Inject tags after example tags. # Config files for agent versions 6.11 and earlier: diff --git a/test/compile_and_run_test.sh b/test/compile_and_run_test.sh index 4e6cb8d..c6ddfbb 100644 --- a/test/compile_and_run_test.sh +++ b/test/compile_and_run_test.sh @@ -29,7 +29,7 @@ compileAndRunVersion() assertCaptured "Installing Datadog runner" export HOME=${BUILD_DIR} - export DD_TAGS="sampletag1:sample,sametag2:sample, sampletag3 sampletag4" + export DD_TAGS="sampletag1:sample,sametag2:sample, sampletag3 sampletag4" chmod +x ${BUILDPACK_HOME}/extra/datadog.sh capture ${BUILDPACK_HOME}/extra/datadog.sh assertFileNotContains "error while loading shared libraries" ${STD_ERR} @@ -37,7 +37,7 @@ compileAndRunVersion() assertCaptured "Starting Datadog Agent" assertCaptured "Starting Datadog Trace Agent" assertNotCaptured "The Datadog Agent has been disabled" - assertCaptured "[DEBUG] Buildpack normalized tags: sampletag1:sample,sametag2:sample,sampletag3,sampletag4" + assertCaptured "[DEBUG] Buildpack normalized tags: sampletag1:sample sametag2:sample sampletag3 sampletag4" } From 47d1ee358b599e4a7afa123c6f595253c499dc60 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 17 Jun 2020 15:18:51 +0200 Subject: [PATCH 006/175] Prepare release 1.18 --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c46058..c8348dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased & outstanding issues] - Non-https repo url and apt fetching +## [1.18] - 2020-06-17 + +### Added +- Datadog agent pinned versions are now `6.20.2` and `7.20.2` + ## [1.17] - 2020-05-04 ### Added From 12b8008940d3891d053e656083b79f82eeff1f12 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 17 Jun 2020 19:42:51 +0200 Subject: [PATCH 007/175] Move version tag --- extra/datadog.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index 7e48212..6c24103 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -45,7 +45,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="dev" +BUILDPACKVERSION="1.18" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" if [ -n "$HEROKU_APP_NAME" ]; then From e3239614e9e65bbe61839c3830c905d995f3c4da Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Thu, 18 Jun 2020 09:43:58 +0200 Subject: [PATCH 008/175] Move buildpackversion back to dev --- extra/datadog.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index 6c24103..7e48212 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -45,7 +45,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="1.18" +BUILDPACKVERSION="dev" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" if [ -n "$HEROKU_APP_NAME" ]; then From bf77f061307c5c8f916ba6a945572fc407da06f8 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Thu, 2 Jul 2020 13:14:50 +0200 Subject: [PATCH 009/175] Add host type as dyno --- extra/datadog.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/extra/datadog.sh b/extra/datadog.sh index 7e48212..51f2122 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -173,6 +173,9 @@ sed -i "s/^# - role:database$/# - role:database\n$DD_TAGS_YAML/" "$DATADOG_C # Agent versions 6.12 and later: sed -i "s/^\(## @param tags\)/$DD_TAGS_YAML\n\1/" "$DATADOG_CONF" +# Export host type as dyno +export DD_HEROKU_DYNO="true" + # Execute the final run logic. if [ -n "$DISABLE_DATADOG_AGENT" ]; then echo "The Datadog Agent has been disabled. Unset the DISABLE_DATADOG_AGENT or set missing environment variables." From 7196f96120b8a7946ae67eabb614b8fce70c1d7f Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Mon, 13 Jul 2020 17:40:51 +0200 Subject: [PATCH 010/175] fix(agent-wrapper): allow to pass multiple command Running `agent-wrapper check foobar` does not work currently and only print help. --- extra/agent-wrapper | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/extra/agent-wrapper b/extra/agent-wrapper index 35b649f..4787971 100644 --- a/extra/agent-wrapper +++ b/extra/agent-wrapper @@ -1,4 +1,6 @@ #!/usr/bin/env bash -# Run the Datadog Agent with the selected command -bash -c "PYTHONPATH=\"$DD_PYTHONPATH\" LD_LIBRARY_PATH=\"$DD_LD_LIBRARY_PATH\" $DD_BIN_DIR/agent -c $DATADOG_CONF "$@" " +# Run the Datadog Agent with the selected command +export PYTHONPATH="$DD_PYTHONPATH" +export LD_LIBRARY_PATH="$DD_LD_LIBRARY_PATH" +exec "$DD_BIN_DIR"/agent -c "$DATADOG_CONF" $* From 54627a1f6ee0e0494530c45fe73e8d1d1e228630 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Thu, 20 Aug 2020 09:55:14 +0200 Subject: [PATCH 011/175] Add tests for python2 regression in agent 7 --- test/compile_and_run_test.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/compile_and_run_test.sh b/test/compile_and_run_test.sh index c6ddfbb..defee57 100644 --- a/test/compile_and_run_test.sh +++ b/test/compile_and_run_test.sh @@ -38,6 +38,8 @@ compileAndRunVersion() assertCaptured "Starting Datadog Trace Agent" assertNotCaptured "The Datadog Agent has been disabled" assertCaptured "[DEBUG] Buildpack normalized tags: sampletag1:sample sametag2:sample sampletag3 sampletag4" + assertNotCaptured "ModuleNotFoundError" + assertNotCaptured "Fatal Python error" } From aee5600d7858245200b72f3bc2e9a6cdb56d8b92 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Thu, 20 Aug 2020 10:39:23 +0200 Subject: [PATCH 012/175] Attemp to remove python2 even on agent 7, to avoid regressions --- bin/compile | 68 +++++++++++++++++++++++++---------------------------- 1 file changed, 32 insertions(+), 36 deletions(-) diff --git a/bin/compile b/bin/compile index 5ace375..e773b3a 100644 --- a/bin/compile +++ b/bin/compile @@ -76,15 +76,24 @@ fi # Prior to Agent 6.14 there was only 1 python version DD_AGENT_BASE_VERSION="6.14" if [ "$DD_AGENT_VERSION" == "$(echo -e "$DD_AGENT_BASE_VERSION\n$DD_AGENT_VERSION" | sort -V | head -n1)" ]; then - NUM_PYTHON="1" + DD_PYTHON_VERSION="2" else - NUM_PYTHON="2" + if [ -f "$ENV_DIR/DD_PYTHON_VERSION" ]; then + DD_PYTHON_VERSION=$(cat "$ENV_DIR/DD_PYTHON_VERSION") + if [ "$DD_PYTHON_VERSION" != "2" ] && [ "$DD_PYTHON_VERSION" != "3" ]; then + topic "ERROR: Wrong Python version: \"$DD_PYTHON_VERSION\"." + echo "Set DD_PYTHON_VERSION to either 2 or 3." | indent + exit 1 + fi + else + DD_PYTHON_VERSION="2" # if not specified, we default to Python 2 + fi fi # Agent 7 onwards is Python3 only and repo file is different DD_AGENT_BASE_VERSION="7" if [ "$DD_AGENT_VERSION" != "$(echo -e "$DD_AGENT_BASE_VERSION\n$DD_AGENT_VERSION" | sort -V | head -n1)" ]; then - NUM_PYTHON="1" + DD_PYTHON_VERSION="3" # Modify repo file to point to agent 7 cp "$APT_REPO_FILE_7" "$APT_REPO_FILE" fi @@ -164,41 +173,28 @@ else rm -rf "$APT_DIR/opt/datadog-agent/embedded/bin/process-agent" fi -# If we have 2 python versions, we remove the one that wasn't selected -if [ "$NUM_PYTHON" = "2" ]; then - if [ -f "$ENV_DIR/DD_PYTHON_VERSION" ]; then - DD_PYTHON_VERSION=$(cat "$ENV_DIR/DD_PYTHON_VERSION") - if [ "$DD_PYTHON_VERSION" != "2" ] && [ "$DD_PYTHON_VERSION" != "3" ]; then - topic "ERROR: Wrong Python version: \"$DD_PYTHON_VERSION\"." - echo "Set DD_PYTHON_VERSION to either 2 or 3." | indent - exit 1 - fi - else - DD_PYTHON_VERSION="2" # if not specified, we default to Python 2 - fi - - # We remove the unneeded version of Python - if [ "$DD_PYTHON_VERSION" = "2" ]; then - topic "*********************************** WARNING ************************************" - echo "Python 2 will be deprecated soon. Agent 7.x ships with Python 3 only." | indent - echo "If you don't run custom checks or those are Python 3 ready, you can" | indent - echo "move to Python 3 now by setting DD_PYTHON_VERSION to 3 and compiling your slug." | indent - echo "********************************************************************************" | indent - rm -rf "$APT_DIR"/opt/datadog-agent/embedded/lib/python3.* - rm -rf "$APT_DIR"/opt/datadog-agent/embedded/lib/libpython3* - rm "$APT_DIR"/opt/datadog-agent/embedded/bin/pip3* - rm "$APT_DIR"/opt/datadog-agent/embedded/bin/python3* - rm "$APT_DIR"/opt/datadog-agent/embedded/bin/pydoc3* - fi - if [ "$DD_PYTHON_VERSION" = "3" ]; then - rm -rf "$APT_DIR"/opt/datadog-agent/embedded/lib/python2.* - rm -rf "$APT_DIR"/opt/datadog-agent/embedded/lib/libpython2* - rm "$APT_DIR"/opt/datadog-agent/embedded/bin/pip2* - rm "$APT_DIR"/opt/datadog-agent/embedded/bin/python2* - rm "$APT_DIR"/opt/datadog-agent/embedded/bin/pydoc* - fi +# We remove the unneeded version of Python +if [ "$DD_PYTHON_VERSION" = "2" ]; then + topic "*********************************** WARNING ************************************" + echo "Python 2 will be deprecated soon. Agent 7.x ships with Python 3 only." | indent + echo "If you don't run custom checks or those are Python 3 ready, you can" | indent + echo "move to Python 3 now by setting DD_PYTHON_VERSION to 3 and compiling your slug." | indent + echo "********************************************************************************" | indent + rm -rf "$APT_DIR"/opt/datadog-agent/embedded/lib/python3.* || true + rm -rf "$APT_DIR"/opt/datadog-agent/embedded/lib/libpython3* || true + rm -f "$APT_DIR"/opt/datadog-agent/embedded/bin/pip3* || true + rm -f "$APT_DIR"/opt/datadog-agent/embedded/bin/python3* || true + rm -f "$APT_DIR"/opt/datadog-agent/embedded/bin/pydoc3* || true +fi +if [ "$DD_PYTHON_VERSION" = "3" ]; then + rm -rf "$APT_DIR"/opt/datadog-agent/embedded/lib/python2.* || true + rm -rf "$APT_DIR"/opt/datadog-agent/embedded/lib/libpython2* || true + rm -f "$APT_DIR"/opt/datadog-agent/embedded/bin/pip2* || true + rm -f "$APT_DIR"/opt/datadog-agent/embedded/bin/python2* || true + rm -f "$APT_DIR"/opt/datadog-agent/embedded/bin/pydoc* || true fi + # Rewrite package-config files find "$APT_DIR" -type f -ipath '*/pkgconfig/*.pc' | xargs --no-run-if-empty -n 1 sed -i -e 's!^prefix=\(.*\)$!prefix='"$APT_DIR"'\1!g' From 7cc4f20452f71139cd2da088be993d5e2696575e Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Fri, 21 Aug 2020 16:30:51 +0200 Subject: [PATCH 013/175] Allow changing DD_VERSION in prerun script --- extra/datadog.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index 51f2122..a8bb1a1 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -160,6 +160,7 @@ else DD_TAGS="$DYNO_TAGS" fi +export DD_VERSION="$DD_VERSION" export DD_TAGS="$DD_TAGS" if [ "$DD_LOG_LEVEL_LOWER" == "debug" ]; then echo "[DEBUG] Buildpack normalized tags: $DD_TAGS_NORMALIZED" @@ -181,11 +182,11 @@ if [ -n "$DISABLE_DATADOG_AGENT" ]; then echo "The Datadog Agent has been disabled. Unset the DISABLE_DATADOG_AGENT or set missing environment variables." else # Get the Agent version number - DD_VERSION="$(expr "$(bash -c "LD_LIBRARY_PATH=\"$DD_LD_LIBRARY_PATH\" $DD_BIN_DIR/agent version")" : 'Agent \([0-9]\+\.[0-9]\+.[0-9]\+\)')" + DATADOG_VERSION="$(expr "$(bash -c "LD_LIBRARY_PATH=\"$DD_LD_LIBRARY_PATH\" $DD_BIN_DIR/agent version")" : 'Agent \([0-9]\+\.[0-9]\+.[0-9]\+\)')" # Prior to Agent 6.4.1, the command is "start" RUN_VERSION="6.4.1" - if [ "$DD_VERSION" == "$(echo -e "$RUN_VERSION\n$DD_VERSION" | sort -V | head -n1)" ]; then + if [ "$DATADOG_VERSION" == "$(echo -e "$RUN_VERSION\n$DATADOG_VERSION" | sort -V | head -n1)" ]; then RUN_COMMAND="start" else RUN_COMMAND="run" From 4b7e1f22f65324e3b7cb04a0ad789da0f0def68a Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Fri, 21 Aug 2020 17:00:02 +0200 Subject: [PATCH 014/175] Add documentation related to DD_VERSION --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1774421..37688e4 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,7 @@ In addition to the environment variables shown above, there are a number of othe | `DD_HOSTNAME` | *Optional.* **WARNING**: Setting the hostname manually may result in metrics continuity errors. It is recommended that you do *not* set this variable. Because dyno hosts are ephemeral it is recommended that you monitor based on the tags `dynoname` or `appname`. | | `DD_DYNO_HOST` | *Optional.* Set to `true` to use the dyno name (e.g. `web.1` or `run.1234`) as the hostname. See the [hostname section](#hostname) below for more information. Defaults to `false` | | `DD_TAGS` | *Optional.* Sets additional tags provided as a space-separated string (**Note**: comma-separated string in buildpack versions `1.16` and earlier; this is still supported to maintain backward compatibility). For example, `heroku config:set DD_TAGS="simple-tag-0 tag-key-1:tag-value-1"`. The buildpack automatically adds the tags `dyno` which represent the dyno name (e.g. web.1) and `dynotype` (the type of dyno, e.g `run` or `web`). See the ["Guide to tagging"][8] for more information. | +| `DD_VERSION` | *Optional.* Sets the version of your application, used to organize traces by version. | | `DD_HISTOGRAM_PERCENTILES` | *Optional.* Optionally set additional percentiles for your histogram metrics. See [How to graph percentiles][9]. | | `DISABLE_DATADOG_AGENT` | *Optional.* When set, the Datadog Agent does not run. | | `DD_APM_ENABLED` | *Optional.* Trace collection is enabled by default. Set this to `false` to disable trace collection. Changing this option requires recompilation of the slug. Check [the upgrading and slug recompilation section](#upgrading-and-slug-recompilation) for details. | @@ -142,7 +143,7 @@ instances: ## Prerun script -In addition to all of the configurations above, you can include a prerun script, `/datadog/prerun.sh`, in your application. The prerun script will run after all of the standard configuration actions and immediately before starting the Datadog Agent. This allows you to modify the environment variables, perform additional configurations, or even disable the Datadog Agent programmatically. +In addition to all of the configurations above, you can include a prerun script, `/datadog/prerun.sh`, in your application. The prerun script will run after all of the standard configuration actions and immediately before starting the Datadog Agent. This allows you to modify the environment variables (for example, DD_TAGS or DD_VERSION), perform additional configurations, or even disable the Datadog Agent programmatically. The example below demonstrates a few of the things you can do in the `prerun.sh` script: @@ -154,6 +155,11 @@ if [ "$DYNOTYPE" == "run" ]; then DISABLE_DATADOG_AGENT="true" fi +# Set app version based on HEROKU_SLUG_COMMIT +if [ -n "$HEROKU_SLUG_COMMIT" ]; then + DD_VERSION=$HEROKU_SLUG_COMMIT +fi + # Update the Postgres configuration from above using the Heroku application environment variable if [ -n "$DATABASE_URL" ]; then POSTGREGEX='^postgres://([^:]+):([^@]+)@([^:]+):([^/]+)/(.*)$' From 01983269c6653dc378a4b243a33388a85a92198c Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Fri, 21 Aug 2020 15:54:21 +0200 Subject: [PATCH 015/175] CHANGELOG for 1.19 --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c8348dc..f9644c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased & outstanding issues] - Non-https repo url and apt fetching +## [1.19] - 2020-08-XX + +### Changed +- Agent now reports `datadog.heroku_agent.running` metric instead of `datadog.agent.running` for agent versions `6/7.22` and newer, to help customers identify which agents are reporting from a Heroku dyno. + +### Fixed +- Buildpack now works correctly with agent versions 7.21.x + +### Added +- Datadog agent pinned versions are now `6.22.0` and `7.22.0` +- DD_VERSION can now be set as part of the `prerun.sh` script + ## [1.18] - 2020-06-17 ### Added From 3f2f1ed5dddf541031c2366e1410243e0b4bce21 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Thu, 27 Aug 2020 22:25:47 +0200 Subject: [PATCH 016/175] Ping 6.22.0 and 7.22.0 agent versions --- bin/compile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/compile b/bin/compile index e773b3a..c5480b4 100644 --- a/bin/compile +++ b/bin/compile @@ -11,8 +11,8 @@ set -o pipefail # set -x # Set agent pinned version -DD_AGENT_PINNED_VERSION_6="6.20.2-1" -DD_AGENT_PINNED_VERSION_7="7.20.2-1" +DD_AGENT_PINNED_VERSION_6="6.22.0-1" +DD_AGENT_PINNED_VERSION_7="7.22.0-1" # Parse and derive params BUILD_DIR=$1 From dc148b83389bc8237904bdd16a3c193ca2592124 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Thu, 27 Aug 2020 22:27:59 +0200 Subject: [PATCH 017/175] Update release date --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9644c9..17ac45a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased & outstanding issues] - Non-https repo url and apt fetching -## [1.19] - 2020-08-XX +## [1.19] - 2020-08-27 ### Changed - Agent now reports `datadog.heroku_agent.running` metric instead of `datadog.agent.running` for agent versions `6/7.22` and newer, to help customers identify which agents are reporting from a Heroku dyno. From f0d67b60338e758caf8a0fcc53b7f4858072fb69 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Thu, 27 Aug 2020 22:29:06 +0200 Subject: [PATCH 018/175] update buildpack version --- extra/datadog.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index a8bb1a1..eb2a394 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -45,7 +45,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="dev" +BUILDPACKVERSION="1.19" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" if [ -n "$HEROKU_APP_NAME" ]; then From 8990f3762155b9ee31174500d8b2be8deedb6429 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Thu, 27 Aug 2020 23:25:08 +0200 Subject: [PATCH 019/175] buildpack version back to dev for master --- extra/datadog.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index eb2a394..a8bb1a1 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -45,7 +45,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="1.19" +BUILDPACKVERSION="dev" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" if [ -n "$HEROKU_APP_NAME" ]; then From 6168c84441858e17dd1887b3563da846cc5d682c Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Tue, 8 Sep 2020 10:40:03 +0200 Subject: [PATCH 020/175] Remove security-agent. Add tests for size increases --- bin/compile | 4 ++++ test/compile_and_run_test.sh | 13 ++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/bin/compile b/bin/compile index c5480b4..e8b34ce 100644 --- a/bin/compile +++ b/bin/compile @@ -155,6 +155,10 @@ done # Remove the system-probe binary, only needed for network performance monitoring rm -rf "$APT_DIR/opt/datadog-agent/embedded/bin/system-probe" +rm -rf "$APT_DIR"/opt/datadog-agent/embedded/lib/libbcc* + +# Remove the Security agent +rm -rf "$APT_DIR/opt/datadog-agent/embedded/bin/security-agent" if [ -f "$ENV_DIR/DD_APM_ENABLED" ]; then DD_APM_ENABLED=$(cat "$ENV_DIR/DD_APM_ENABLED") diff --git a/test/compile_and_run_test.sh b/test/compile_and_run_test.sh index defee57..78e06f1 100644 --- a/test/compile_and_run_test.sh +++ b/test/compile_and_run_test.sh @@ -21,6 +21,7 @@ compileAndRunVersion() { echo $1 > "${ENV_DIR}/DD_AGENT_VERSION" + echo "true" > "${ENV_DIR}/DD_PROCESS_AGENT" echo "Testing Datadog Agent version $1" compile assertCaptured "Installing dependencies" @@ -41,6 +42,16 @@ compileAndRunVersion() assertNotCaptured "ModuleNotFoundError" assertNotCaptured "Fatal Python error" + + SIZE_BIN=$(du -s $HOME/.apt/opt/datadog-agent/bin | cut -f1) + SIZE_LIB=$(du -s $HOME/.apt/opt/datadog-agent/embedded/lib | cut -f1) + SIZE_EMB_BIN=$(du -s $HOME/.apt/opt/datadog-agent/embedded/bin | cut -f1) + + assertTrue "Binary folder is too big: ${SIZE_BIN}" "[ $SIZE_BIN -lt 105000 ]" + assertTrue "Embedded library folder is too big: ${SIZE_LIB}" "[ $SIZE_LIB -lt 230000 ]" + assertTrue "Embedded binary folder is too big: ${SIZE_EMB_BIN}" "[ $SIZE_EMB_BIN -lt 115000 ]" + + rm -rf $HOME/.apt/ } testReleased6Versions() @@ -49,7 +60,7 @@ testReleased6Versions() for VERSION in ${AGENT_VERSIONS}; do compileAndRunVersion $VERSION - done + done } testReleased7Versions() From 6561e8493b70d70983d175adca850ad148c6ea86 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 9 Sep 2020 13:05:23 +0200 Subject: [PATCH 021/175] Only run the agent binary test for versions 6.13 onwards --- test/compile_and_run_test.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/compile_and_run_test.sh b/test/compile_and_run_test.sh index 78e06f1..ebef261 100644 --- a/test/compile_and_run_test.sh +++ b/test/compile_and_run_test.sh @@ -47,7 +47,11 @@ compileAndRunVersion() SIZE_LIB=$(du -s $HOME/.apt/opt/datadog-agent/embedded/lib | cut -f1) SIZE_EMB_BIN=$(du -s $HOME/.apt/opt/datadog-agent/embedded/bin | cut -f1) - assertTrue "Binary folder is too big: ${SIZE_BIN}" "[ $SIZE_BIN -lt 105000 ]" + # Prior to 6.13 the agent was too big + BIGSIZE_VERSION="6.13.0-1" + if test "$BIGSIZE_VERSION" = "$(echo "$BIGSIZE_VERSION\n$1" | sort -V | head -n1)" ; then + assertTrue "Binary folder is too big: ${SIZE_BIN}" "[ $SIZE_BIN -lt 105000 ]" + fi assertTrue "Embedded library folder is too big: ${SIZE_LIB}" "[ $SIZE_LIB -lt 230000 ]" assertTrue "Embedded binary folder is too big: ${SIZE_EMB_BIN}" "[ $SIZE_EMB_BIN -lt 115000 ]" From 973828d4cb638bbc7494fd04a9e537d9e79afd08 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 9 Sep 2020 15:54:45 +0200 Subject: [PATCH 022/175] Fix the log_config run path even if logs are not enabled --- extra/datadog.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index a8bb1a1..4f8a097 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -64,10 +64,8 @@ if [ "$DD_PROCESS_AGENT" == "true" ]; then fi # Set the right path for the log collector -if [ "$DD_LOGS_ENABLED" == "true" ]; then - sed -i -e"s|^# logs_config:$|logs_config:|" "$DATADOG_CONF" - sed -i -e"s|^logs_config:$|logs_config:\n run_path: $DD_RUN_DIR|" "$DATADOG_CONF" -fi +sed -i -e"s|^# logs_config:$|logs_config:|" "$DATADOG_CONF" +sed -i -e"s|^logs_config:$|logs_config:\n run_path: $DD_RUN_DIR|" "$DATADOG_CONF" # For a list of env vars to override datadog.yaml, see: # https://github.com/DataDog/datadog-agent/blob/master/pkg/config/config.go#L145 From 68010f8de1cb75d756916a8f167057f1025ee319 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 9 Sep 2020 16:46:20 +0200 Subject: [PATCH 023/175] Prepare 1.20 release --- CHANGELOG.md | 8 ++++++++ extra/datadog.sh | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 17ac45a..180e886 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased & outstanding issues] - Non-https repo url and apt fetching +## [1.20] - 2020-09-09 + +### Changed +- Removed the security-agent and system-probe dependencies to reduce slug size + +### Fixed +- version-history.json is now created successfully, even if DD_LOGS_ENABLED="false" + ## [1.19] - 2020-08-27 ### Changed diff --git a/extra/datadog.sh b/extra/datadog.sh index 4f8a097..eb7d257 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -45,7 +45,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="dev" +BUILDPACKVERSION="1.20" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" if [ -n "$HEROKU_APP_NAME" ]; then From 94f8da0a86c6446855cc97cd697af8f6f0784767 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 9 Sep 2020 17:22:11 +0200 Subject: [PATCH 024/175] Set buildpack version back to dev --- extra/datadog.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index eb7d257..4f8a097 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -45,7 +45,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="1.20" +BUILDPACKVERSION="dev" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" if [ -n "$HEROKU_APP_NAME" ]; then From 942c8af0ef689c6fe915aaa101308241e1e0490d Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Sat, 12 Sep 2020 16:13:27 +0100 Subject: [PATCH 025/175] Change buildpack detection name from "Sysstat" to "Datadog" Since whilst debugging a customer support ticket, it was confusing to see a buildpack called "Sysstat" in the build log output, which did not appear to match any of the buildpacks in the buildpacks URLs list for that app. The buildpack is for Datadog, so calling it that seems more intuitive, plus it then matches the GitHub repo URL that appears in the buildpacks URLs list. Example build log before: ``` -----> Deleting 1 files matching .slugignore patterns. -----> Sysstat app detected -----> Updating apt caches for dependencies ... -----> Ruby app detected ... ``` Example build log after: ``` -----> Deleting 1 files matching .slugignore patterns. -----> Datadog app detected -----> Updating apt caches for dependencies ... -----> Ruby app detected ... ``` --- CHANGELOG.md | 1 + bin/detect | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 180e886..f70af7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased & outstanding issues] - Non-https repo url and apt fetching +- Buildpack name reported by `bin/detect` is now "Datadog" instead of "Sysstat" ## [1.20] - 2020-09-09 diff --git a/bin/detect b/bin/detect index 343199d..183373a 100644 --- a/bin/detect +++ b/bin/detect @@ -1,4 +1,4 @@ #!/usr/bin/env bash # bin/detect -echo "Sysstat" && exit 0 +echo "Datadog" && exit 0 From 6a1271a83c05b77464bd020682606afdaf542e68 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Mon, 21 Sep 2020 10:31:48 +0200 Subject: [PATCH 026/175] Change run_path config entry --- extra/datadog.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index 4f8a097..69429d9 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -26,9 +26,9 @@ DD_PROC_LOG="$DD_LOG_DIR/datadog-proc.log" # Move Datadog config files into place cp "$DATADOG_CONF.example" "$DATADOG_CONF" -# Update the Datadog conf yaml with the correct conf.d and checks.d +# Update the Datadog conf yaml with the correct conf.d and checks.d and the correct run path sed -i -e"s|^.*confd_path:.*$|confd_path: $DD_CONF_DIR/conf.d|" "$DATADOG_CONF" -sed -i -e"s|^.*additional_checksd:.*$|additional_checksd: $DD_CONF_DIR/checks.d|" "$DATADOG_CONF" +sed -i -e"s|^.*additional_checksd:.*$|additional_checksd: $DD_CONF_DIR/checks.d\nrun_path: $DD_RUN_DIR|" "$DATADOG_CONF" # Include application's datadog configs APP_DATADOG="/app/datadog" From e7c1e6f3e65f20c566384381ef7c4b8ae58bc370 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Mon, 21 Sep 2020 12:10:11 +0200 Subject: [PATCH 027/175] Clarify buildpack layers order --- README.md | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 37688e4..4c344f2 100644 --- a/README.md +++ b/README.md @@ -19,21 +19,13 @@ heroku buildpacks:add heroku/ruby heroku labs:enable runtime-dyno-metadata -a $(heroku apps:info|grep ===|cut -d' ' -f2) # Add this buildpack and set your Datadog API key -heroku buildpacks:add https://github.com/DataDog/heroku-buildpack-datadog.git# +heroku buildpacks:add --index 1 https://github.com/DataDog/heroku-buildpack-datadog.git# heroku config:add DD_API_KEY= # Deploy to Heroku git push heroku master ``` -**Warning**: Buildpacks that install apt packages (e.g. [apt][3], [puppeteer dependencies][4]) or buildpacks that modify the `/app` folder (e.g. [monorepo][5]) need to be added *before* the Datadog buildpack. For example, if your application uses the `ruby`, `datadog` and `apt` buildpacks, this would be a correct `heroku buildpacks` output: - -```text -1. heroku/ruby -2. https://github.com/heroku/heroku-buildpack-apt.git -3. https://github.com/DataDog/heroku-buildpack-datadog.git -``` - Replace `` with your [Datadog API key][6]. Replace `` with the [Buildpack release][7] you want to use. @@ -41,6 +33,16 @@ Once complete, the Datadog Agent is started automatically when each dyno starts. The Datadog Agent provides a listening port on `8125` for statsd/dogstatsd metrics and events. Traces are collected on port `8126`. +
+Warning: The last buildpack in the list will be used to determine the process type for the application. Also, buildpacks that install apt packages (e.g. [apt][3], [puppeteer dependencies][4]) or buildpacks that modify the `/app` folder (e.g. [monorepo][5]) need to be added *before* the Datadog buildpack. For example, if your application uses the `ruby`, `datadog` and `apt` buildpacks, this would be a correct `heroku buildpacks` output: + +```text +1. https://github.com/heroku/heroku-buildpack-apt.git +2. https://github.com/DataDog/heroku-buildpack-datadog.git +3. heroku/ruby +``` +
+ ## Upgrading and slug recompilation Upgrading this buildpack or modifying certain buildpack options requires you to clear your application's build cache and recompile your slug. From 02306770feef226dcda693ce8f69b9077741c48d Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Thu, 8 Oct 2020 17:25:05 +0200 Subject: [PATCH 028/175] Pin agent 7.23 and 6.23 --- bin/compile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/compile b/bin/compile index e8b34ce..3989ca7 100644 --- a/bin/compile +++ b/bin/compile @@ -11,8 +11,8 @@ set -o pipefail # set -x # Set agent pinned version -DD_AGENT_PINNED_VERSION_6="6.22.0-1" -DD_AGENT_PINNED_VERSION_7="7.22.0-1" +DD_AGENT_PINNED_VERSION_6="6.23.0-1" +DD_AGENT_PINNED_VERSION_7="7.23.0-1" # Parse and derive params BUILD_DIR=$1 From ca46d3c440fe506c93722d29621f104eb4036155 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Fri, 9 Oct 2020 12:13:02 +0200 Subject: [PATCH 029/175] Prepare 1.21 release --- CHANGELOG.md | 7 +++++++ extra/datadog.sh | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f70af7d..af63bb6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased & outstanding issues] - Non-https repo url and apt fetching + +## [1.21] - 2020-10-09 + +### Added +- Datadog agent pinned versions are now `6.23.0` and `7.23.0` + +### Change - Buildpack name reported by `bin/detect` is now "Datadog" instead of "Sysstat" ## [1.20] - 2020-09-09 diff --git a/extra/datadog.sh b/extra/datadog.sh index 69429d9..888b910 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -45,7 +45,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="dev" +BUILDPACKVERSION="1.21" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" if [ -n "$HEROKU_APP_NAME" ]; then From 8c5669bd695788811431d99884da48b85d0d3fc6 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Fri, 9 Oct 2020 12:51:05 +0200 Subject: [PATCH 030/175] Revert buildpack version to dev --- extra/datadog.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index 888b910..69429d9 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -45,7 +45,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="1.21" +BUILDPACKVERSION="dev" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" if [ -n "$HEROKU_APP_NAME" ]; then From 46903f39c1a4b306557e5d7a4026a334e0ffc727 Mon Sep 17 00:00:00 2001 From: Kylian Serrania Date: Mon, 26 Oct 2020 12:23:57 +0100 Subject: [PATCH 031/175] Update Agent GPG keys --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4c344f2..6b37fd2 100644 --- a/README.md +++ b/README.md @@ -222,9 +222,10 @@ As an example, if you are building your Docker image using a Debian based OS, yo RUN apt-get update \ && apt-get install -y gpg apt-transport-https gpg-agent curl ca-certificates -# Add Datadog repository and signing key +# Add Datadog repository and signing keys RUN sh -c "echo 'deb https://apt.datadoghq.com/ stable 7' > /etc/apt/sources.list.d/datadog.list" RUN apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 A2923DFF56EDA6E76E55E492D3A80E30382E94DE +RUN apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 D75CEA17048B9ACBF186794B32637D44F14F620E # Install the Datadog agent RUN apt-get update && apt-get -y --force-yes install --reinstall datadog-agent From e9616a622fff824570dc1ae0e32d8cbb84e5a81b Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Mon, 26 Oct 2020 17:14:05 +0100 Subject: [PATCH 032/175] Disable cloud provider metadata --- extra/datadog.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/extra/datadog.sh b/extra/datadog.sh index 69429d9..58f7317 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -30,6 +30,9 @@ cp "$DATADOG_CONF.example" "$DATADOG_CONF" sed -i -e"s|^.*confd_path:.*$|confd_path: $DD_CONF_DIR/conf.d|" "$DATADOG_CONF" sed -i -e"s|^.*additional_checksd:.*$|additional_checksd: $DD_CONF_DIR/checks.d\nrun_path: $DD_RUN_DIR|" "$DATADOG_CONF" +# Update the Datadog conf yaml to disable cloud provider metadata +sed -i -e"s|^.*cloud_provider_metadata:.*$|cloud_provider_metadata: []|" "$DATADOG_CONF" + # Include application's datadog configs APP_DATADOG="/app/datadog" APP_DATADOG_CONF_DIR="$APP_DATADOG/conf.d" From 814df315efd1257591890ded5b9acab56074bcb2 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Mon, 26 Oct 2020 17:32:05 +0100 Subject: [PATCH 033/175] Add a install_info file to Heroku --- extra/datadog.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/extra/datadog.sh b/extra/datadog.sh index 69429d9..f15b675 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -8,6 +8,7 @@ DD_RUN_DIR="$DD_DIR/run" export DD_BIN_DIR="$DD_DIR/bin/agent" DD_LOG_DIR="$APT_DIR/var/log/datadog" DD_CONF_DIR="$APT_DIR/etc/datadog-agent" +DD_INSTALL_INFO="$DD_CONF_DIR/install_info" export DATADOG_CONF="$DD_CONF_DIR/datadog.yaml" # Update Env Vars with new paths for apt packages @@ -48,6 +49,9 @@ DYNOTYPE=${DYNO%%.*} BUILDPACKVERSION="dev" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" +# Include install method +echo -e "install_method:\n tool: heroku\n tool_version: heroku\n installer_version: heroku-$BUILDPACKVERSION" > "$DD_INSTALL_INFO" + if [ -n "$HEROKU_APP_NAME" ]; then DYNO_TAGS="$DYNO_TAGS appname:$HEROKU_APP_NAME" fi From 79288cabb09b64e371c40031a00f189f60ecf19c Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Thu, 5 Nov 2020 15:55:58 +0100 Subject: [PATCH 034/175] Remove installing dependencies --- bin/compile | 8 -------- 1 file changed, 8 deletions(-) diff --git a/bin/compile b/bin/compile index 3989ca7..86d1d2b 100644 --- a/bin/compile +++ b/bin/compile @@ -40,14 +40,6 @@ mkdir -p "$APT_DIR" topic "Updating apt caches for dependencies" apt-get $APT_OPTIONS update | indent -topic "Installing dependencies" -DEPS="libpci-dev libpci3 libsensors4 libsensors4-dev libsnmp-base libsnmp30" -apt-get $APT_OPTIONS -y --force-yes -d install --reinstall --no-install-recommends $DEPS | indent -IFS=" " read -a DEP_PKGS <<< "$DEPS" -for DEP in ${DEP_PKGS[@]}; do - echo "Installing $DEP" | indent - ls -t "$APT_CACHE_DIR"/archives/"$DEP"\_*.deb | head -1 | xargs -i dpkg -x '{}' "$APT_DIR" -done # Install GPG key topic "Install gpg key for Datadog APT Repository" From ce29f32a25d909f635f60997b22887117e063583 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Thu, 5 Nov 2020 16:20:43 +0100 Subject: [PATCH 035/175] Fix tests --- test/compile_and_run_test.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/test/compile_and_run_test.sh b/test/compile_and_run_test.sh index ebef261..d6067a5 100644 --- a/test/compile_and_run_test.sh +++ b/test/compile_and_run_test.sh @@ -24,7 +24,6 @@ compileAndRunVersion() echo "true" > "${ENV_DIR}/DD_PROCESS_AGENT" echo "Testing Datadog Agent version $1" compile - assertCaptured "Installing dependencies" assertCaptured "Downloading Datadog Agent $1" assertCaptured "Installing Datadog Agent" assertCaptured "Installing Datadog runner" From 8deebdaaac37946a740f56d9e3d198e073d40e1a Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 11 Nov 2020 16:27:05 +0100 Subject: [PATCH 036/175] Clean up the apt cache if stack changes --- bin/compile | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/bin/compile b/bin/compile index 3989ca7..b33670e 100644 --- a/bin/compile +++ b/bin/compile @@ -20,6 +20,17 @@ CACHE_DIR=$2 ENV_DIR=$3 BUILDPACK_DIR=$(cd "$(dirname "$0")"; cd ..; pwd) +# Store which STACK we are running on in the cache to bust the cache if it changes +if [ -f $CACHE_DIR/.apt/STACK ]; then + CACHED_STACK=$(cat "$CACHE_DIR/.apt/STACK") +else + CACHED_STACK=$STACK +fi + +# Ensure we store the STACK in the cache for next time. +mkdir -p "$CACHE_DIR/.apt" +echo "$STACK" > "$CACHE_DIR/.apt/STACK" + # Load formating tools source "$BUILDPACK_DIR/bin/common.sh" @@ -36,6 +47,17 @@ mkdir -p "$APT_CACHE_DIR/archives/partial" mkdir -p "$APT_STATE_DIR/lists/partial" mkdir -p "$APT_DIR" +if [[ $CACHED_STACK == $STACK ]] ; then + # STACK has not changed, reusing cache + topic "Reusing cache" +else + # STACK changed, clean up APT cache + topic "Detected Stack changes, flushing cache" + rm -rf $APT_CACHE_DIR + mkdir -p "$APT_CACHE_DIR/archives/partial" + mkdir -p "$APT_STATE_DIR/lists/partial" +fi + # Install dependencies topic "Updating apt caches for dependencies" apt-get $APT_OPTIONS update | indent From 18b7421f5cf8dd31b713a39b60c2c5b716abdf52 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 9 Dec 2020 10:12:12 +0100 Subject: [PATCH 037/175] Pin 6.24 and 7.24 agent versions --- bin/compile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/compile b/bin/compile index c4f55cf..9789ee5 100644 --- a/bin/compile +++ b/bin/compile @@ -11,8 +11,8 @@ set -o pipefail # set -x # Set agent pinned version -DD_AGENT_PINNED_VERSION_6="6.23.0-1" -DD_AGENT_PINNED_VERSION_7="7.23.0-1" +DD_AGENT_PINNED_VERSION_6="6.24.0-1" +DD_AGENT_PINNED_VERSION_7="7.24.0-1" # Parse and derive params BUILD_DIR=$1 From 04ec7bc1be64b268e1976c8d3fd9990dc4c6569b Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 9 Dec 2020 13:14:52 +0100 Subject: [PATCH 038/175] Prepare 1.22 release --- CHANGELOG.md | 9 +++++++++ extra/datadog.sh | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index af63bb6..7ecea88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased & outstanding issues] - Non-https repo url and apt fetching +## [1.22] - 2020-12-09 + +### Added +- Datadog agent pinned versions are now `6.24.0` and `7.24.0` +- Heroku APT cache is now cleaned up when changing stacks + +### Removed +- Removed Datadog agent unneeded dependencies + ## [1.21] - 2020-10-09 ### Added diff --git a/extra/datadog.sh b/extra/datadog.sh index 144c471..73f76f8 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -49,7 +49,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="dev" +BUILDPACKVERSION="1.22" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # Include install method From 2a5b7d7e62185a62c38c7bf6fd86e25fb172956e Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 9 Dec 2020 15:39:45 +0100 Subject: [PATCH 039/175] Revert buildpack version to dev --- extra/datadog.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index 73f76f8..144c471 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -49,7 +49,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="1.22" +BUILDPACKVERSION="dev" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # Include install method From 70c82779e908fa17196051e277dd37721850d842 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Tue, 22 Dec 2020 15:37:43 +0100 Subject: [PATCH 040/175] Update changelog to mention heroku-20 support --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ecea88..4cccb74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added - Datadog agent pinned versions are now `6.24.0` and `7.24.0` - Heroku APT cache is now cleaned up when changing stacks +- Heroku-20 stack is now supported ### Removed - Removed Datadog agent unneeded dependencies From 276fdf624939b05ce181b3e0fc3da36c23b84320 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Tue, 22 Dec 2020 16:23:11 +0100 Subject: [PATCH 041/175] Update README to follow Heroku best practices for buildpacks Update install instructions to recommend not pinning the buildpack --- README.md | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 6b37fd2..a959fa7 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ heroku buildpacks:add heroku/ruby heroku labs:enable runtime-dyno-metadata -a $(heroku apps:info|grep ===|cut -d' ' -f2) # Add this buildpack and set your Datadog API key -heroku buildpacks:add --index 1 https://github.com/DataDog/heroku-buildpack-datadog.git# +heroku buildpacks:add --index 1 https://github.com/DataDog/heroku-buildpack-datadog.git heroku config:add DD_API_KEY= # Deploy to Heroku @@ -27,13 +27,11 @@ git push heroku master ``` Replace `` with your [Datadog API key][6]. -Replace `` with the [Buildpack release][7] you want to use. Once complete, the Datadog Agent is started automatically when each dyno starts. The Datadog Agent provides a listening port on `8125` for statsd/dogstatsd metrics and events. Traces are collected on port `8126`. -
Warning: The last buildpack in the list will be used to determine the process type for the application. Also, buildpacks that install apt packages (e.g. [apt][3], [puppeteer dependencies][4]) or buildpacks that modify the `/app` folder (e.g. [monorepo][5]) need to be added *before* the Datadog buildpack. For example, if your application uses the `ruby`, `datadog` and `apt` buildpacks, this would be a correct `heroku buildpacks` output: ```text @@ -41,7 +39,18 @@ Warning: The last buildpack in the list will be used to determine the process ty 2. https://github.com/DataDog/heroku-buildpack-datadog.git 3. heroku/ruby ``` -
+ +## Pinning a specific buildpack version and a specific Datadog agent version + +Heroku recommends to always use the latest commit of a buildpack. If you need to pin the buildpack version, you can do so by specifying the buildpack release tag: + +``` +heroku buildpacks:add --index 1 https://github.com/DataDog/heroku-buildpack-datadog.git# +``` + +Replace `` with the [Buildpack release][7] you want to use. + +In both cases, the version of the Datadog agent can be pinned as well by setting `DD_AGENT_VERSION` environment variable. ## Upgrading and slug recompilation From 1b9a3013d5d2a95ebdebc49c90ebcf25629ef650 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Tue, 22 Dec 2020 17:21:02 +0100 Subject: [PATCH 042/175] Restore warning, using standard links --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a959fa7..e0bf0f4 100644 --- a/README.md +++ b/README.md @@ -32,13 +32,15 @@ Once complete, the Datadog Agent is started automatically when each dyno starts. The Datadog Agent provides a listening port on `8125` for statsd/dogstatsd metrics and events. Traces are collected on port `8126`. -Warning: The last buildpack in the list will be used to determine the process type for the application. Also, buildpacks that install apt packages (e.g. [apt][3], [puppeteer dependencies][4]) or buildpacks that modify the `/app` folder (e.g. [monorepo][5]) need to be added *before* the Datadog buildpack. For example, if your application uses the `ruby`, `datadog` and `apt` buildpacks, this would be a correct `heroku buildpacks` output: +
+Warning: The last buildpack in the list will be used to determine the process type for the application. Also, buildpacks that install apt packages (e.g. apt, puppeteer dependencies) or buildpacks that modify the `/app` folder (e.g. monorepo) need to be added *before* the Datadog buildpack. For example, if your application uses the `ruby`, `datadog` and `apt` buildpacks, this would be a correct `heroku buildpacks` output: ```text 1. https://github.com/heroku/heroku-buildpack-apt.git 2. https://github.com/DataDog/heroku-buildpack-datadog.git 3. heroku/ruby ``` +
## Pinning a specific buildpack version and a specific Datadog agent version From a97454fafb90bd52d80f295bab38bcfbed02669e Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Tue, 22 Dec 2020 17:26:32 +0100 Subject: [PATCH 043/175] Reorganize links --- README.md | 77 ++++++++++++++++++++++++++----------------------------- 1 file changed, 37 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index e0bf0f4..e4e549e 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ heroku config:add DD_API_KEY= git push heroku master ``` -Replace `` with your [Datadog API key][6]. +Replace `` with your [Datadog API key][3]. Once complete, the Datadog Agent is started automatically when each dyno starts. @@ -50,7 +50,7 @@ Heroku recommends to always use the latest commit of a buildpack. If you need to heroku buildpacks:add --index 1 https://github.com/DataDog/heroku-buildpack-datadog.git# ``` -Replace `` with the [Buildpack release][7] you want to use. +Replace `` with the [Buildpack release][4] you want to use. In both cases, the version of the Datadog agent can be pinned as well by setting `DD_AGENT_VERSION` environment variable. @@ -92,9 +92,9 @@ In addition to the environment variables shown above, there are a number of othe | `DD_API_KEY` | *Required.* Your API key is available from the [Datadog API Integrations][6] page. Note that this is the *API* key, not the application key. | | `DD_HOSTNAME` | *Optional.* **WARNING**: Setting the hostname manually may result in metrics continuity errors. It is recommended that you do *not* set this variable. Because dyno hosts are ephemeral it is recommended that you monitor based on the tags `dynoname` or `appname`. | | `DD_DYNO_HOST` | *Optional.* Set to `true` to use the dyno name (e.g. `web.1` or `run.1234`) as the hostname. See the [hostname section](#hostname) below for more information. Defaults to `false` | -| `DD_TAGS` | *Optional.* Sets additional tags provided as a space-separated string (**Note**: comma-separated string in buildpack versions `1.16` and earlier; this is still supported to maintain backward compatibility). For example, `heroku config:set DD_TAGS="simple-tag-0 tag-key-1:tag-value-1"`. The buildpack automatically adds the tags `dyno` which represent the dyno name (e.g. web.1) and `dynotype` (the type of dyno, e.g `run` or `web`). See the ["Guide to tagging"][8] for more information. | +| `DD_TAGS` | *Optional.* Sets additional tags provided as a space-separated string (**Note**: comma-separated string in buildpack versions `1.16` and earlier; this is still supported to maintain backward compatibility). For example, `heroku config:set DD_TAGS="simple-tag-0 tag-key-1:tag-value-1"`. The buildpack automatically adds the tags `dyno` which represent the dyno name (e.g. web.1) and `dynotype` (the type of dyno, e.g `run` or `web`). See the ["Guide to tagging"][5] for more information. | | `DD_VERSION` | *Optional.* Sets the version of your application, used to organize traces by version. | -| `DD_HISTOGRAM_PERCENTILES` | *Optional.* Optionally set additional percentiles for your histogram metrics. See [How to graph percentiles][9]. | +| `DD_HISTOGRAM_PERCENTILES` | *Optional.* Optionally set additional percentiles for your histogram metrics. See [How to graph percentiles][6]. | | `DISABLE_DATADOG_AGENT` | *Optional.* When set, the Datadog Agent does not run. | | `DD_APM_ENABLED` | *Optional.* Trace collection is enabled by default. Set this to `false` to disable trace collection. Changing this option requires recompilation of the slug. Check [the upgrading and slug recompilation section](#upgrading-and-slug-recompilation) for details. | | `DD_PROCESS_AGENT` | *Optional.* The Datadog Process Agent is disabled by default. Set this to `true` to enable the Process Agent. Changing this option requires recompilation of the slug. Check [the upgrading and slug recompilation section](#upgrading-and-slug-recompilation) for details. | @@ -104,7 +104,7 @@ In addition to the environment variables shown above, there are a number of othe | `DD_DISABLE_HOST_METRICS` | *Optional.* By default, the buildpack reports system metrics for the host machine running the dyno. Set this to `true` to disable system metrics collection. See the [system metrics section](#system-metrics) below for more information. | | `DD_PYTHON_VERSION` | *Optional.* Starting with version `6.14.0`, Datadog Agent ships with Python versions `2` and `3`. The buildpack will only keep one of the versions. Set this to `2` or `3` to select the Python version you want the agent to keep. If not set, the buildpack will keep `2`. Check the [Python versions section](#python-and-agent-versions) for more information. Changing this option requires recompiling the slug. Check [the upgrading and slug recompilation section](#upgrading-and-slug-recompilation) for details. | -For additional documentation, refer to the [Datadog Agent documentation][10]. +For additional documentation, refer to the [Datadog Agent documentation][7]. ## Hostname @@ -112,7 +112,7 @@ Heroku dynos are ephemeral—they can move to different host machines whenever n Depending on your use case, you may want to set your hostname so that hosts are aggregated and report a lower number. To do this, Set `DD_DYNO_HOST` to `true`. This will cause the Agent to report the hostname as the app and dyno name (e.g. `appname.web.1` or `appname.run.1234`) and your host count will closely match your dyno usage. One drawback is that you may see some metrics continuity errors whenever a dyno is cycled. -For this to work correctly, `HEROKU_APP_NAME` needs to be set. The easiest way to do this is by [enabling dyno metadata][11] Take into account that dyno metadata is not yet available in Private Spaces, in which case you will need to set `HEROKU_APP_NAME` manually. +For this to work correctly, `HEROKU_APP_NAME` needs to be set. The easiest way to do this is by [enabling dyno metadata][8] Take into account that dyno metadata is not yet available in Private Spaces, in which case you will need to set `HEROKU_APP_NAME` manually. ## System metrics @@ -120,9 +120,9 @@ By default, the buildpack collects system metrics for the host machine running y In order to collect system metrics for your dynos, you must: -1. Enable the [Heroku Labs: log-runtime-metrics][12]. -2. Use the [Datadog log drain][13] to collect metric logs from the Heroku Logplex and forward them to Datadog. -3. Generate [log-based metric][14] over the collected logs. +1. Enable the [Heroku Labs: log-runtime-metrics][9]. +2. Use the [Datadog log drain][10] to collect metric logs from the Heroku Logplex and forward them to Datadog. +3. Generate [log-based metric][11] over the collected logs. ## File locations @@ -132,9 +132,9 @@ In order to collect system metrics for your dynos, you must: ## Enabling integrations -To enable a [Datadog- integration][15], create a file `/datadog/conf.d/.yaml` in the root of your application. During the dyno start up, your YAML files are copied to the appropriate Datadog Agent configuration directories. +To enable a [Datadog- integration][12], create a file `/datadog/conf.d/.yaml` in the root of your application. During the dyno start up, your YAML files are copied to the appropriate Datadog Agent configuration directories. -For example, to enable the [Datadog-Redis integration][16], create the file `/datadog/conf.d/redisdb.yaml` at the root of your application: +For example, to enable the [Datadog-Redis integration][13], create the file `/datadog/conf.d/redisdb.yaml` at the root of your application: ```yaml init_config: @@ -152,7 +152,7 @@ instances: port: 6379 ``` -**Note**: See the sample [redisdb.d/conf.yaml][17] for all available configuration options. +**Note**: See the sample [redisdb.d/conf.yaml][14] for all available configuration options. ## Prerun script @@ -204,7 +204,7 @@ To reduce your slug size, make sure that `DD_APM_ENABLED` is set to `false`, if ## Debugging -To run any of the information/debugging commands listed in the [Agent's documentation][18] use the `agent-wrapper` command. +To run any of the information/debugging commands listed in the [Agent's documentation][15] use the `agent-wrapper` command. For example, to display the status of your Datadog Agent and enabled integrations, run: @@ -220,11 +220,11 @@ Agent v7 only ships with Python version `3`. If you are not using custom checks ## Heroku log collection -The Datadog buildpack does not collect logs from the Heroku platform. To set up Heroku log collection, see the [dedicated guide][13]. +The Datadog buildpack does not collect logs from the Heroku platform. To set up Heroku log collection, see the [dedicated guide][9]. ## Using Heroku with Docker images -This buildpack only works for Heroku deployments that use [Heroku's Slug Compiler][19]. If you are deploying your application in Heroku using Docker containers you will need to add the Datadog agent as part of your Docker image and start the agent as a different process in your container. +This buildpack only works for Heroku deployments that use [Heroku's Slug Compiler][16]. If you are deploying your application in Heroku using Docker containers you will need to add the Datadog agent as part of your Docker image and start the agent as a different process in your container. As an example, if you are building your Docker image using a Debian based OS, you will need to add the following lines to your `Dockerfile`: @@ -263,15 +263,15 @@ datadog-agent run & /opt/datadog-agent/embedded/bin/process-agent --config=/etc/datadog-agent/datadog.yaml ``` -For more advanced options in the Docker image, reference the [Datadog Agent Docker files][20]. +For more advanced options in the Docker image, reference the [Datadog Agent Docker files][17]. ## Contributing -See the [contributing documentation][21] to learn how to open an issue or PR to the [Heroku-buildpack-datadog repository][22]. +See the [contributing documentation][18] to learn how to open an issue or PR to the [Heroku-buildpack-datadog repository][19]. ## History -Earlier versions of this project were forked from the [miketheman heroku-buildpack-datadog project][23]. It was largely rewritten for Datadog's Agent version 6. Changes and more information can be found in the [changelog][24]. +Earlier versions of this project were forked from the [miketheman heroku-buildpack-datadog project][20]. It was largely rewritten for Datadog's Agent version 6. Changes and more information can be found in the [changelog][21]. ## FAQs / Troubleshooting @@ -285,25 +285,22 @@ After an upgrade of the buildpack or agent, you must clear your build cache and [1]: https://devcenter.heroku.com/articles/buildpacks [2]: https://docs.datadoghq.com/libraries -[3]: https://github.com/heroku/heroku-buildpack-apt -[4]: https://github.com/jontewks/puppeteer-heroku-buildpack -[5]: https://github.com/lstoll/heroku-buildpack-monorepo -[6]: https://app.datadoghq.com/account/settings#api -[7]: https://github.com/DataDog/heroku-buildpack-datadog/releases -[8]: https://docs.datadoghq.com/tagging/ -[9]: https://docs.datadoghq.com/dashboards/guide/how-to-graph-percentiles-in-datadog/ -[10]: https://docs.datadoghq.com/agent -[11]: https://devcenter.heroku.com/articles/dyno-metadata -[12]: https://devcenter.heroku.com/articles/log-runtime-metrics -[13]: https://docs.datadoghq.com/logs/guide/collect-heroku-logs -[14]: https://docs.datadoghq.com/logs/logs_to_metrics/ -[15]: https://docs.datadoghq.com/integrations/ -[16]: https://docs.datadoghq.com/integrations/redisdb/ -[17]: https://github.com/DataDog/integrations-core/blob/master/redisdb/datadog_checks/redisdb/data/conf.yaml.example -[18]: https://docs.datadoghq.com/agent/guide/agent-commands/#agent-status-and-information -[19]: https://devcenter.heroku.com/articles/slug-compiler -[20]: https://github.com/DataDog/datadog-agent/tree/master/Dockerfiles -[21]: https://github.com/DataDog/heroku-buildpack-datadog/blob/master/CONTRIBUTING.md -[22]: https://github.com/DataDog/heroku-buildpack-datadog -[23]: https://github.com/miketheman/heroku-buildpack-datadog -[24]: https://github.com/DataDog/heroku-buildpack-datadog/blob/master/CHANGELOG.md +[3]: https://app.datadoghq.com/account/settings#api +[4]: https://github.com/DataDog/heroku-buildpack-datadog/releases +[5]: https://docs.datadoghq.com/tagging/ +[6]: https://docs.datadoghq.com/dashboards/guide/how-to-graph-percentiles-in-datadog/ +[7]: https://docs.datadoghq.com/agent +[8]: https://devcenter.heroku.com/articles/dyno-metadata +[9]: https://devcenter.heroku.com/articles/log-runtime-metrics +[10]: https://docs.datadoghq.com/logs/guide/collect-heroku-logs +[11]: https://docs.datadoghq.com/logs/logs_to_metrics/ +[12]: https://docs.datadoghq.com/integrations/ +[13]: https://docs.datadoghq.com/integrations/redisdb/ +[14]: https://github.com/DataDog/integrations-core/blob/master/redisdb/datadog_checks/redisdb/data/conf.yaml.example +[15]: https://docs.datadoghq.com/agent/guide/agent-commands/#agent-status-and-information +[16]: https://devcenter.heroku.com/articles/slug-compiler +[17]: https://github.com/DataDog/datadog-agent/tree/master/Dockerfiles +[18]: https://github.com/DataDog/heroku-buildpack-datadog/blob/master/CONTRIBUTING.md +[19]: https://github.com/DataDog/heroku-buildpack-datadog +[20]: https://github.com/miketheman/heroku-buildpack-datadog +[21]: https://github.com/DataDog/heroku-buildpack-datadog/blob/master/CHANGELOG.md From b8d610e8bbe5034017f7c7b05ad5cc2090e1a3af Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Tue, 22 Dec 2020 17:28:00 +0100 Subject: [PATCH 044/175] Change wording of pinned versions --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e4e549e..6726caa 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ heroku buildpacks:add --index 1 https://github.com/DataDog/heroku-buildpack-data Replace `` with the [Buildpack release][4] you want to use. -In both cases, the version of the Datadog agent can be pinned as well by setting `DD_AGENT_VERSION` environment variable. +By default, the buildpack pins the latest version of the Datadog Agent at the time of release. You can pin the Agent to an earlier version by setting the `DD_AGENT_VERSION` environment variable. ## Upgrading and slug recompilation From 634fa426ba841e8e3c144016234d86ad481ac1aa Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Fri, 15 Jan 2021 10:30:51 +0100 Subject: [PATCH 045/175] Clean build cache when agent version changes --- bin/compile | 51 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/bin/compile b/bin/compile index 9789ee5..54a1bc1 100644 --- a/bin/compile +++ b/bin/compile @@ -31,6 +31,32 @@ fi mkdir -p "$CACHE_DIR/.apt" echo "$STACK" > "$CACHE_DIR/.apt/STACK" +# If a version hasn't been specified, use the pinned version +if [ -f "$ENV_DIR/DD_AGENT_VERSION" ]; then + DD_AGENT_VERSION=$(cat "$ENV_DIR/DD_AGENT_VERSION") +else + if [ -f "$ENV_DIR/DD_AGENT_MAJOR_VERSION" ]; then + DD_AGENT_MAJOR_VERSION=$(cat "$ENV_DIR/DD_AGENT_MAJOR_VERSION") + if [ "$DD_AGENT_MAJOR_VERSION" == "7" ]; then + DD_AGENT_VERSION="$DD_AGENT_PINNED_VERSION_7" + else + DD_AGENT_VERSION="$DD_AGENT_PINNED_VERSION_6" + fi + else + DD_AGENT_VERSION="$DD_AGENT_PINNED_VERSION_6" + fi +fi + +# Store which Datadog agent we deployed in the previous compile to bust the cache if it changes +if [ -f $CACHE_DIR/.apt/DD_AGENT_VERSION ]; then + CACHED_DD_AGENT_VERSION=$(cat "$CACHE_DIR/.apt/DD_AGENT_VERSION") +else + CACHED_DD_AGENT_VERSION=$DD_AGENT_VERSION +fi + +# Ensure we store the agent version in the cache for next time. +echo "$DD_AGENT_VERSION" > "$CACHE_DIR/.apt/DD_AGENT_VERSION" + # Load formating tools source "$BUILDPACK_DIR/bin/common.sh" @@ -47,12 +73,12 @@ mkdir -p "$APT_CACHE_DIR/archives/partial" mkdir -p "$APT_STATE_DIR/lists/partial" mkdir -p "$APT_DIR" -if [[ $CACHED_STACK == $STACK ]] ; then - # STACK has not changed, reusing cache - topic "Reusing cache" +if [[ $CACHED_STACK == $STACK ]] && [[ $CACHED_DD_AGENT_VERSION == $DD_AGENT_VERSION ]]; then + # STACK nor DD_AGENT_VERSION have changed, reusing cache + topic "Stack version and agent version haven't changed, reusing cache" else - # STACK changed, clean up APT cache - topic "Detected Stack changes, flushing cache" + # STACK or DD_AGENT_VERSION changed, clean up APT cache + topic "Detected Stack and/or agent version changes, flushing cache" rm -rf $APT_CACHE_DIR mkdir -p "$APT_CACHE_DIR/archives/partial" mkdir -p "$APT_STATE_DIR/lists/partial" @@ -71,21 +97,6 @@ GPG_HOME_DIR="$BUILD_DIR/.gnupg" mkdir -p "$GPG_HOME_DIR" gpg --ignore-time-conflict --no-options --no-default-keyring --homedir "$GPG_HOME_DIR" --keyring "$APT_KEYRING" --import "$GPG_KEY_FILE" | indent -# If a version hasn't been specified, use the pinned version -if [ -f "$ENV_DIR/DD_AGENT_VERSION" ]; then - DD_AGENT_VERSION=$(cat "$ENV_DIR/DD_AGENT_VERSION") -else - if [ -f "$ENV_DIR/DD_AGENT_MAJOR_VERSION" ]; then - DD_AGENT_MAJOR_VERSION=$(cat "$ENV_DIR/DD_AGENT_MAJOR_VERSION") - if [ "$DD_AGENT_MAJOR_VERSION" == "7" ]; then - DD_AGENT_VERSION="$DD_AGENT_PINNED_VERSION_7" - else - DD_AGENT_VERSION="$DD_AGENT_PINNED_VERSION_6" - fi - else - DD_AGENT_VERSION="$DD_AGENT_PINNED_VERSION_6" - fi -fi # Prior to Agent 6.14 there was only 1 python version DD_AGENT_BASE_VERSION="6.14" From 37525b9e653c91d5a5fcc0284bbd76d87859177d Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Fri, 15 Jan 2021 15:03:39 +0100 Subject: [PATCH 046/175] Move agent pinned version to 6.25 and 7.25 --- bin/compile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/compile b/bin/compile index 9789ee5..1b8a83d 100644 --- a/bin/compile +++ b/bin/compile @@ -11,8 +11,8 @@ set -o pipefail # set -x # Set agent pinned version -DD_AGENT_PINNED_VERSION_6="6.24.0-1" -DD_AGENT_PINNED_VERSION_7="7.24.0-1" +DD_AGENT_PINNED_VERSION_6="6.25.0-1" +DD_AGENT_PINNED_VERSION_7="7.25.0-1" # Parse and derive params BUILD_DIR=$1 From 44e05ded1983e49735509b2ba27d7b6ff93b99c9 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 20 Jan 2021 15:42:03 +0100 Subject: [PATCH 047/175] Prepare release 1.23 --- CHANGELOG.md | 6 ++++++ extra/datadog.sh | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4cccb74..77543cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased & outstanding issues] - Non-https repo url and apt fetching +## [1.23] - 2021-01-20 + +### Added +- Datadog agent pinned versions are now `6.25.0` and `7.25.0` +- Heroku APT cache is now cleaned up automatically when DD_AGENT_VERSION changes + ## [1.22] - 2020-12-09 ### Added diff --git a/extra/datadog.sh b/extra/datadog.sh index 144c471..f6a62b5 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -49,7 +49,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="dev" +BUILDPACKVERSION="1.23" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # Include install method From e2af27739132e03231f07b8884f50766a6dcb54b Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 20 Jan 2021 16:40:23 +0100 Subject: [PATCH 048/175] Revert buildpack version to dev --- extra/datadog.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index f6a62b5..144c471 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -49,7 +49,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="1.23" +BUILDPACKVERSION="dev" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # Include install method From d5c0667c48df89476da1f562cc3be9cdd41563eb Mon Sep 17 00:00:00 2001 From: Ian Vaughan Date: Fri, 22 Jan 2021 14:32:29 +0000 Subject: [PATCH 049/175] Allow override of datadog config path The default config folder path for configuration files is /datadog, but users may wish to use a different location. One use case is that a Rails application has a standard folder layout, and system/infra folders tend to be dot prefixed, eg .github This change adds a check that if DD_HEROKU_CONF_FOLDER is set, it will use that path, otherwise it falls back to normal default of /datadog. --- README.md | 1 + extra/datadog.sh | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6726caa..0750cb3 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,7 @@ In addition to the environment variables shown above, there are a number of othe | `DD_AGENT_MAJOR_VERSION` | *Optional.* By default, the buildpack installs the latest 6.x version of the Datadog Agent available in the package repository. Set this variable to `7` to install the latest 7.x version of the Datadog Agent. Check the [Python versions section](#python-and-agent-versions) for more information on the relation of the agent version and Python version. Changing this option requires recompiling the slug. Check [the upgrading and slug recompilation section](#upgrading-and-slug-recompilation) for details. | | `DD_DISABLE_HOST_METRICS` | *Optional.* By default, the buildpack reports system metrics for the host machine running the dyno. Set this to `true` to disable system metrics collection. See the [system metrics section](#system-metrics) below for more information. | | `DD_PYTHON_VERSION` | *Optional.* Starting with version `6.14.0`, Datadog Agent ships with Python versions `2` and `3`. The buildpack will only keep one of the versions. Set this to `2` or `3` to select the Python version you want the agent to keep. If not set, the buildpack will keep `2`. Check the [Python versions section](#python-and-agent-versions) for more information. Changing this option requires recompiling the slug. Check [the upgrading and slug recompilation section](#upgrading-and-slug-recompilation) for details. | +| `DD_HEROKU_CONF_FOLDER` | *Optional.* By default, the buildpack will look in the root of your application for a folder `/datadog` for any configuration files you wish to include, eg see [prerun.sh script](#prerun-script). This location can be overridden by setting this to your desired path. | For additional documentation, refer to the [Datadog Agent documentation][7]. diff --git a/extra/datadog.sh b/extra/datadog.sh index 144c471..e9379e1 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -35,7 +35,8 @@ sed -i -e"s|^.*additional_checksd:.*$|additional_checksd: $DD_CONF_DIR/checks.d\ sed -i -e"s|^.*cloud_provider_metadata:.*$|cloud_provider_metadata: []|" "$DATADOG_CONF" # Include application's datadog configs -APP_DATADOG="/app/datadog" +APP_DATADOG_DEFAULT="/app/datadog" +APP_DATADOG="${DD_HEROKU_CONF_FOLDER:=$APP_DATADOG_DEFAULT}" APP_DATADOG_CONF_DIR="$APP_DATADOG/conf.d" for file in "$APP_DATADOG_CONF_DIR"/*.yaml; do From c147ed3a7dfd51b1d27b4d0a97e790a13db3dd17 Mon Sep 17 00:00:00 2001 From: Ian Vaughan Date: Fri, 29 Jan 2021 18:32:17 +0000 Subject: [PATCH 050/175] Add to usage of /datadog with info on DD_HEROKU_CONF_FOLDER --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0750cb3..87117d5 100644 --- a/README.md +++ b/README.md @@ -133,9 +133,9 @@ In order to collect system metrics for your dynos, you must: ## Enabling integrations -To enable a [Datadog- integration][12], create a file `/datadog/conf.d/.yaml` in the root of your application. During the dyno start up, your YAML files are copied to the appropriate Datadog Agent configuration directories. +To enable a [Datadog- integration][12], create a file in the datadog configuration folder within your application. During the dyno start up, your YAML files are copied to the appropriate Datadog Agent configuration directories. -For example, to enable the [Datadog-Redis integration][13], create the file `/datadog/conf.d/redisdb.yaml` at the root of your application: +For example, to enable the [Datadog-Redis integration][13], add the file `/datadog/conf.d/redisdb.yaml` at the root of your application (or `/$DD_HEROKU_CONF_FOLDER/conf.d/redisdb.yaml` if you have changed this [configuration option](#configuration)): ```yaml init_config: From 27cc878aa031d44fef04d3210339e494df287658 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Fri, 29 Jan 2021 16:02:35 +0100 Subject: [PATCH 051/175] Add documentation about installing community integrations --- README.md | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 87117d5..ab62e2c 100644 --- a/README.md +++ b/README.md @@ -155,9 +155,21 @@ instances: **Note**: See the sample [redisdb.d/conf.yaml][14] for all available configuration options. +If the integration you are enabling is part of the [Community Integrations][22], you will also need to install the package as part of the [prerun script](#prerun-script). + +``` +agent-wrapper integration install -t datadog-== +``` + +For example, to install the [ping integration][23], create the configuration file `datadog/conf.d/ping.yaml` and add the following line to your prerun script: + +``` +agent-wrapper integration install -t datadog-ping==1.0.0 +``` + ## Prerun script -In addition to all of the configurations above, you can include a prerun script, `/datadog/prerun.sh`, in your application. The prerun script will run after all of the standard configuration actions and immediately before starting the Datadog Agent. This allows you to modify the environment variables (for example, DD_TAGS or DD_VERSION), perform additional configurations, or even disable the Datadog Agent programmatically. +In addition to all of the configurations above, you can include a prerun script, `/datadog/prerun.sh`, in your application. The prerun script will run after all of the standard configuration actions and immediately before starting the Datadog Agent. This allows you to modify the environment variables (for example, DD_TAGS or DD_VERSION), perform additional configurations, install community integrations, or even disable the Datadog Agent programmatically. The example below demonstrates a few of the things you can do in the `prerun.sh` script: @@ -185,6 +197,9 @@ if [ -n "$DATABASE_URL" ]; then sed -i "s//${BASH_REMATCH[5]}/" "$DD_CONF_DIR/conf.d/postgres.d/conf.yaml" fi fi + +# Install the "ping" community integration +agent-wrapper integration install -t datadog-ping==1.0.0 ``` ## Limiting Datadog's console output @@ -305,3 +320,5 @@ After an upgrade of the buildpack or agent, you must clear your build cache and [19]: https://github.com/DataDog/heroku-buildpack-datadog [20]: https://github.com/miketheman/heroku-buildpack-datadog [21]: https://github.com/DataDog/heroku-buildpack-datadog/blob/master/CHANGELOG.md +[22]: https://github.com/DataDog/integrations-extras/ +[23]: https://github.com/DataDog/integrations-extras/tree/master/ping From f2dfd02dec6e44340253e11758f9c508d37f624a Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Tue, 2 Feb 2021 13:07:41 +0100 Subject: [PATCH 052/175] Reorganize links --- README.md | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index ab62e2c..a59e6c2 100644 --- a/README.md +++ b/README.md @@ -155,13 +155,13 @@ instances: **Note**: See the sample [redisdb.d/conf.yaml][14] for all available configuration options. -If the integration you are enabling is part of the [Community Integrations][22], you will also need to install the package as part of the [prerun script](#prerun-script). +If the integration you are enabling is part of the [Community Integrations][15], you will also need to install the package as part of the [prerun script](#prerun-script). ``` agent-wrapper integration install -t datadog-== ``` -For example, to install the [ping integration][23], create the configuration file `datadog/conf.d/ping.yaml` and add the following line to your prerun script: +For example, to install the [ping integration][16], create the configuration file `datadog/conf.d/ping.yaml` and add the following line to your prerun script: ``` agent-wrapper integration install -t datadog-ping==1.0.0 @@ -220,7 +220,7 @@ To reduce your slug size, make sure that `DD_APM_ENABLED` is set to `false`, if ## Debugging -To run any of the information/debugging commands listed in the [Agent's documentation][15] use the `agent-wrapper` command. +To run any of the information/debugging commands listed in the [Agent's documentation][17] use the `agent-wrapper` command. For example, to display the status of your Datadog Agent and enabled integrations, run: @@ -240,7 +240,7 @@ The Datadog buildpack does not collect logs from the Heroku platform. To set up ## Using Heroku with Docker images -This buildpack only works for Heroku deployments that use [Heroku's Slug Compiler][16]. If you are deploying your application in Heroku using Docker containers you will need to add the Datadog agent as part of your Docker image and start the agent as a different process in your container. +This buildpack only works for Heroku deployments that use [Heroku's Slug Compiler][18]. If you are deploying your application in Heroku using Docker containers you will need to add the Datadog agent as part of your Docker image and start the agent as a different process in your container. As an example, if you are building your Docker image using a Debian based OS, you will need to add the following lines to your `Dockerfile`: @@ -279,15 +279,15 @@ datadog-agent run & /opt/datadog-agent/embedded/bin/process-agent --config=/etc/datadog-agent/datadog.yaml ``` -For more advanced options in the Docker image, reference the [Datadog Agent Docker files][17]. +For more advanced options in the Docker image, reference the [Datadog Agent Docker files][19]. ## Contributing -See the [contributing documentation][18] to learn how to open an issue or PR to the [Heroku-buildpack-datadog repository][19]. +See the [contributing documentation][20] to learn how to open an issue or PR to the [Heroku-buildpack-datadog repository][21]. ## History -Earlier versions of this project were forked from the [miketheman heroku-buildpack-datadog project][20]. It was largely rewritten for Datadog's Agent version 6. Changes and more information can be found in the [changelog][21]. +Earlier versions of this project were forked from the [miketheman heroku-buildpack-datadog project][22]. It was largely rewritten for Datadog's Agent version 6. Changes and more information can be found in the [changelog][23]. ## FAQs / Troubleshooting @@ -313,12 +313,12 @@ After an upgrade of the buildpack or agent, you must clear your build cache and [12]: https://docs.datadoghq.com/integrations/ [13]: https://docs.datadoghq.com/integrations/redisdb/ [14]: https://github.com/DataDog/integrations-core/blob/master/redisdb/datadog_checks/redisdb/data/conf.yaml.example -[15]: https://docs.datadoghq.com/agent/guide/agent-commands/#agent-status-and-information -[16]: https://devcenter.heroku.com/articles/slug-compiler -[17]: https://github.com/DataDog/datadog-agent/tree/master/Dockerfiles -[18]: https://github.com/DataDog/heroku-buildpack-datadog/blob/master/CONTRIBUTING.md -[19]: https://github.com/DataDog/heroku-buildpack-datadog -[20]: https://github.com/miketheman/heroku-buildpack-datadog -[21]: https://github.com/DataDog/heroku-buildpack-datadog/blob/master/CHANGELOG.md -[22]: https://github.com/DataDog/integrations-extras/ -[23]: https://github.com/DataDog/integrations-extras/tree/master/ping +[15]: https://github.com/DataDog/integrations-extras/ +[16]: https://github.com/DataDog/integrations-extras/tree/master/ping +[17]: https://docs.datadoghq.com/agent/guide/agent-commands/#agent-status-and-information +[18]: https://devcenter.heroku.com/articles/slug-compiler +[19]: https://github.com/DataDog/datadog-agent/tree/master/Dockerfiles +[20]: https://github.com/DataDog/heroku-buildpack-datadog/blob/master/CONTRIBUTING.md +[21]: https://github.com/DataDog/heroku-buildpack-datadog +[22]: https://github.com/miketheman/heroku-buildpack-datadog +[23]: https://github.com/DataDog/heroku-buildpack-datadog/blob/master/CHANGELOG.md From efda8d200e92212b6a320237a0b7eeac9b4e2ece Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Tue, 9 Feb 2021 11:52:37 +0100 Subject: [PATCH 053/175] Update instructions to clear the Heroku build cache --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a59e6c2..a44e122 100644 --- a/README.md +++ b/README.md @@ -70,13 +70,13 @@ To upgrade this buildpack and/or to change any of these options, for example `DD ```shell # Install the Heroku Repo plugin -heroku plugins:install heroku-repo +heroku plugins:install heroku-builds # Set new version of the Agent heroku config:set DD_AGENT_VERSION= -a appname # Clears Heroku's build cache for "appname" application -heroku repo:purge_cache -a appname +heroku builds:cache:purge -a appname # Rebuild your slug with the new Agent version: git commit --allow-empty -m "Purge cache" From 85e8f9dc52304eefdcef87bd878d801c13dab9f8 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Thu, 11 Feb 2021 10:12:27 +0100 Subject: [PATCH 054/175] Make order of buildpacks more important --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a44e122..51fe566 100644 --- a/README.md +++ b/README.md @@ -32,15 +32,16 @@ Once complete, the Datadog Agent is started automatically when each dyno starts. The Datadog Agent provides a listening port on `8125` for statsd/dogstatsd metrics and events. Traces are collected on port `8126`. -
-Warning: The last buildpack in the list will be used to determine the process type for the application. Also, buildpacks that install apt packages (e.g. apt, puppeteer dependencies) or buildpacks that modify the `/app` folder (e.g. monorepo) need to be added *before* the Datadog buildpack. For example, if your application uses the `ruby`, `datadog` and `apt` buildpacks, this would be a correct `heroku buildpacks` output: +### Order of buildpacks +As explained in [Heroku's documentation](https://devcenter.heroku.com/articles/using-multiple-buildpacks-for-an-app#viewing-buildpacks), the last buildpack in the list will be used to determine the process type for the application. + +Buildpacks that install apt packages (e.g. [heroku-buildpack-apt](https://github.com/heroku/heroku-buildpack-apt), [puppeteer-heroku-buildpack](https://github.com/jontewks/puppeteer-heroku-buildpack)) or buildpacks that modify the `/app` folder (e.g. [heroku-buildpack-monorepo](https://github.com/lstoll/heroku-buildpack-monorepo)) need to be added **before** the Datadog buildpack. For example, if your application uses the `ruby`, `datadog` and `apt` buildpacks, this would be a correct `heroku buildpacks` output: ```text 1. https://github.com/heroku/heroku-buildpack-apt.git 2. https://github.com/DataDog/heroku-buildpack-datadog.git 3. heroku/ruby ``` -
## Pinning a specific buildpack version and a specific Datadog agent version From 7c60b299a858f22f769bd68ff42578856583f48d Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Thu, 11 Feb 2021 10:18:37 +0100 Subject: [PATCH 055/175] Purging the cache is no longer needed --- README.md | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 51fe566..ee0a939 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ By default, the buildpack pins the latest version of the Datadog Agent at the ti ## Upgrading and slug recompilation -Upgrading this buildpack or modifying certain buildpack options requires you to clear your application's build cache and recompile your slug. +Upgrading this buildpack or modifying certain buildpack options requires you to recompile your slug. The following options require a slug recompilation: @@ -70,17 +70,11 @@ The following options require a slug recompilation: To upgrade this buildpack and/or to change any of these options, for example `DD_AGENT_VERSION`, the following steps are required: ```shell -# Install the Heroku Repo plugin -heroku plugins:install heroku-builds - # Set new version of the Agent heroku config:set DD_AGENT_VERSION= -a appname -# Clears Heroku's build cache for "appname" application -heroku builds:cache:purge -a appname - # Rebuild your slug with the new Agent version: -git commit --allow-empty -m "Purge cache" +git commit --allow-empty -m "Rebuild slug" git push heroku master ``` From d60865bf61e87c3787e280b74f3e714ed1af8911 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Thu, 11 Feb 2021 10:22:07 +0100 Subject: [PATCH 056/175] Move links to the bottom --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ee0a939..904e078 100644 --- a/README.md +++ b/README.md @@ -33,9 +33,9 @@ Once complete, the Datadog Agent is started automatically when each dyno starts. The Datadog Agent provides a listening port on `8125` for statsd/dogstatsd metrics and events. Traces are collected on port `8126`. ### Order of buildpacks -As explained in [Heroku's documentation](https://devcenter.heroku.com/articles/using-multiple-buildpacks-for-an-app#viewing-buildpacks), the last buildpack in the list will be used to determine the process type for the application. +As explained in [Heroku's documentation][24], the last buildpack in the list will be used to determine the process type for the application. -Buildpacks that install apt packages (e.g. [heroku-buildpack-apt](https://github.com/heroku/heroku-buildpack-apt), [puppeteer-heroku-buildpack](https://github.com/jontewks/puppeteer-heroku-buildpack)) or buildpacks that modify the `/app` folder (e.g. [heroku-buildpack-monorepo](https://github.com/lstoll/heroku-buildpack-monorepo)) need to be added **before** the Datadog buildpack. For example, if your application uses the `ruby`, `datadog` and `apt` buildpacks, this would be a correct `heroku buildpacks` output: +Buildpacks that install apt packages (e.g. [heroku-buildpack-apt][25], [puppeteer-heroku-buildpack][26]) or buildpacks that modify the `/app` folder (e.g. [heroku-buildpack-monorepo][27]) need to be added **before** the Datadog buildpack. For example, if your application uses the `ruby`, `datadog` and `apt` buildpacks, this would be a correct `heroku buildpacks` output: ```text 1. https://github.com/heroku/heroku-buildpack-apt.git @@ -317,3 +317,7 @@ After an upgrade of the buildpack or agent, you must clear your build cache and [21]: https://github.com/DataDog/heroku-buildpack-datadog [22]: https://github.com/miketheman/heroku-buildpack-datadog [23]: https://github.com/DataDog/heroku-buildpack-datadog/blob/master/CHANGELOG.md +[24]: https://devcenter.heroku.com/articles/using-multiple-buildpacks-for-an-app#viewing-buildpacks +[25]: https://github.com/heroku/heroku-buildpack-apt +[26]: https://github.com/jontewks/puppeteer-heroku-buildpack +[27]: https://github.com/lstoll/heroku-buildpack-monorepo From d88b25006b65799b0a93f1ca8466cb4dfb40f367 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Fri, 19 Feb 2021 16:17:48 +0100 Subject: [PATCH 057/175] Reorganize links --- README.md | 95 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 48 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 904e078..f93d871 100644 --- a/README.md +++ b/README.md @@ -33,9 +33,9 @@ Once complete, the Datadog Agent is started automatically when each dyno starts. The Datadog Agent provides a listening port on `8125` for statsd/dogstatsd metrics and events. Traces are collected on port `8126`. ### Order of buildpacks -As explained in [Heroku's documentation][24], the last buildpack in the list will be used to determine the process type for the application. +As explained in [Heroku's documentation][4], the last buildpack in the list will be used to determine the process type for the application. -Buildpacks that install apt packages (e.g. [heroku-buildpack-apt][25], [puppeteer-heroku-buildpack][26]) or buildpacks that modify the `/app` folder (e.g. [heroku-buildpack-monorepo][27]) need to be added **before** the Datadog buildpack. For example, if your application uses the `ruby`, `datadog` and `apt` buildpacks, this would be a correct `heroku buildpacks` output: +Buildpacks that install apt packages (e.g. [heroku-buildpack-apt][5], [puppeteer-heroku-buildpack][6]) or buildpacks that modify the `/app` folder (e.g. [heroku-buildpack-monorepo][7]) need to be added **before** the Datadog buildpack. For example, if your application uses the `ruby`, `datadog` and `apt` buildpacks, this would be a correct `heroku buildpacks` output: ```text 1. https://github.com/heroku/heroku-buildpack-apt.git @@ -51,7 +51,7 @@ Heroku recommends to always use the latest commit of a buildpack. If you need to heroku buildpacks:add --index 1 https://github.com/DataDog/heroku-buildpack-datadog.git# ``` -Replace `` with the [Buildpack release][4] you want to use. +Replace `` with the [Buildpack release][8] you want to use. By default, the buildpack pins the latest version of the Datadog Agent at the time of release. You can pin the Agent to an earlier version by setting the `DD_AGENT_VERSION` environment variable. @@ -84,12 +84,12 @@ In addition to the environment variables shown above, there are a number of othe | Setting | Description | |----------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `DD_API_KEY` | *Required.* Your API key is available from the [Datadog API Integrations][6] page. Note that this is the *API* key, not the application key. | +| `DD_API_KEY` | *Required.* Your API key is available from the [Datadog API Integrations][9] page. Note that this is the *API* key, not the application key. | | `DD_HOSTNAME` | *Optional.* **WARNING**: Setting the hostname manually may result in metrics continuity errors. It is recommended that you do *not* set this variable. Because dyno hosts are ephemeral it is recommended that you monitor based on the tags `dynoname` or `appname`. | | `DD_DYNO_HOST` | *Optional.* Set to `true` to use the dyno name (e.g. `web.1` or `run.1234`) as the hostname. See the [hostname section](#hostname) below for more information. Defaults to `false` | -| `DD_TAGS` | *Optional.* Sets additional tags provided as a space-separated string (**Note**: comma-separated string in buildpack versions `1.16` and earlier; this is still supported to maintain backward compatibility). For example, `heroku config:set DD_TAGS="simple-tag-0 tag-key-1:tag-value-1"`. The buildpack automatically adds the tags `dyno` which represent the dyno name (e.g. web.1) and `dynotype` (the type of dyno, e.g `run` or `web`). See the ["Guide to tagging"][5] for more information. | +| `DD_TAGS` | *Optional.* Sets additional tags provided as a space-separated string (**Note**: comma-separated string in buildpack versions `1.16` and earlier; this is still supported to maintain backward compatibility). For example, `heroku config:set DD_TAGS="simple-tag-0 tag-key-1:tag-value-1"`. The buildpack automatically adds the tags `dyno` which represent the dyno name (e.g. web.1) and `dynotype` (the type of dyno, e.g `run` or `web`). See the ["Guide to tagging"][10] for more information. | | `DD_VERSION` | *Optional.* Sets the version of your application, used to organize traces by version. | -| `DD_HISTOGRAM_PERCENTILES` | *Optional.* Optionally set additional percentiles for your histogram metrics. See [How to graph percentiles][6]. | +| `DD_HISTOGRAM_PERCENTILES` | *Optional.* Optionally set additional percentiles for your histogram metrics. See [How to graph percentiles][11]. | | `DISABLE_DATADOG_AGENT` | *Optional.* When set, the Datadog Agent does not run. | | `DD_APM_ENABLED` | *Optional.* Trace collection is enabled by default. Set this to `false` to disable trace collection. Changing this option requires recompilation of the slug. Check [the upgrading and slug recompilation section](#upgrading-and-slug-recompilation) for details. | | `DD_PROCESS_AGENT` | *Optional.* The Datadog Process Agent is disabled by default. Set this to `true` to enable the Process Agent. Changing this option requires recompilation of the slug. Check [the upgrading and slug recompilation section](#upgrading-and-slug-recompilation) for details. | @@ -100,7 +100,7 @@ In addition to the environment variables shown above, there are a number of othe | `DD_PYTHON_VERSION` | *Optional.* Starting with version `6.14.0`, Datadog Agent ships with Python versions `2` and `3`. The buildpack will only keep one of the versions. Set this to `2` or `3` to select the Python version you want the agent to keep. If not set, the buildpack will keep `2`. Check the [Python versions section](#python-and-agent-versions) for more information. Changing this option requires recompiling the slug. Check [the upgrading and slug recompilation section](#upgrading-and-slug-recompilation) for details. | | `DD_HEROKU_CONF_FOLDER` | *Optional.* By default, the buildpack will look in the root of your application for a folder `/datadog` for any configuration files you wish to include, eg see [prerun.sh script](#prerun-script). This location can be overridden by setting this to your desired path. | -For additional documentation, refer to the [Datadog Agent documentation][7]. +For additional documentation, refer to the [Datadog Agent documentation][12]. ## Hostname @@ -108,7 +108,7 @@ Heroku dynos are ephemeral—they can move to different host machines whenever n Depending on your use case, you may want to set your hostname so that hosts are aggregated and report a lower number. To do this, Set `DD_DYNO_HOST` to `true`. This will cause the Agent to report the hostname as the app and dyno name (e.g. `appname.web.1` or `appname.run.1234`) and your host count will closely match your dyno usage. One drawback is that you may see some metrics continuity errors whenever a dyno is cycled. -For this to work correctly, `HEROKU_APP_NAME` needs to be set. The easiest way to do this is by [enabling dyno metadata][8] Take into account that dyno metadata is not yet available in Private Spaces, in which case you will need to set `HEROKU_APP_NAME` manually. +For this to work correctly, `HEROKU_APP_NAME` needs to be set. The easiest way to do this is by [enabling dyno metadata][13] Take into account that dyno metadata is not yet available in Private Spaces, in which case you will need to set `HEROKU_APP_NAME` manually. ## System metrics @@ -116,9 +116,9 @@ By default, the buildpack collects system metrics for the host machine running y In order to collect system metrics for your dynos, you must: -1. Enable the [Heroku Labs: log-runtime-metrics][9]. -2. Use the [Datadog log drain][10] to collect metric logs from the Heroku Logplex and forward them to Datadog. -3. Generate [log-based metric][11] over the collected logs. +1. Enable the [Heroku Labs: log-runtime-metrics][14]. +2. Use the [Datadog log drain][15] to collect metric logs from the Heroku Logplex and forward them to Datadog. +3. Generate [log-based metric][16] over the collected logs. ## File locations @@ -128,9 +128,9 @@ In order to collect system metrics for your dynos, you must: ## Enabling integrations -To enable a [Datadog- integration][12], create a file in the datadog configuration folder within your application. During the dyno start up, your YAML files are copied to the appropriate Datadog Agent configuration directories. +To enable a [Datadog- integration][17], create a file in the datadog configuration folder within your application. During the dyno start up, your YAML files are copied to the appropriate Datadog Agent configuration directories. -For example, to enable the [Datadog-Redis integration][13], add the file `/datadog/conf.d/redisdb.yaml` at the root of your application (or `/$DD_HEROKU_CONF_FOLDER/conf.d/redisdb.yaml` if you have changed this [configuration option](#configuration)): +For example, to enable the [Datadog-Redis integration][18], add the file `/datadog/conf.d/redisdb.yaml` at the root of your application (or `/$DD_HEROKU_CONF_FOLDER/conf.d/redisdb.yaml` if you have changed this [configuration option](#configuration)): ```yaml init_config: @@ -148,15 +148,15 @@ instances: port: 6379 ``` -**Note**: See the sample [redisdb.d/conf.yaml][14] for all available configuration options. +**Note**: See the sample [redisdb.d/conf.yaml][19] for all available configuration options. -If the integration you are enabling is part of the [Community Integrations][15], you will also need to install the package as part of the [prerun script](#prerun-script). +If the integration you are enabling is part of the [Community Integrations][20], you will also need to install the package as part of the [prerun script](#prerun-script). ``` agent-wrapper integration install -t datadog-== ``` -For example, to install the [ping integration][16], create the configuration file `datadog/conf.d/ping.yaml` and add the following line to your prerun script: +For example, to install the [ping integration][21], create the configuration file `datadog/conf.d/ping.yaml` and add the following line to your prerun script: ``` agent-wrapper integration install -t datadog-ping==1.0.0 @@ -215,7 +215,7 @@ To reduce your slug size, make sure that `DD_APM_ENABLED` is set to `false`, if ## Debugging -To run any of the information/debugging commands listed in the [Agent's documentation][17] use the `agent-wrapper` command. +To run any of the information/debugging commands listed in the [Agent's documentation][22] use the `agent-wrapper` command. For example, to display the status of your Datadog Agent and enabled integrations, run: @@ -231,11 +231,11 @@ Agent v7 only ships with Python version `3`. If you are not using custom checks ## Heroku log collection -The Datadog buildpack does not collect logs from the Heroku platform. To set up Heroku log collection, see the [dedicated guide][9]. +The Datadog buildpack does not collect logs from the Heroku platform. To set up Heroku log collection, see the [dedicated guide][15]. ## Using Heroku with Docker images -This buildpack only works for Heroku deployments that use [Heroku's Slug Compiler][18]. If you are deploying your application in Heroku using Docker containers you will need to add the Datadog agent as part of your Docker image and start the agent as a different process in your container. +This buildpack only works for Heroku deployments that use [Heroku's Slug Compiler][23]. If you are deploying your application in Heroku using Docker containers you will need to add the Datadog agent as part of your Docker image and start the agent as a different process in your container. As an example, if you are building your Docker image using a Debian based OS, you will need to add the following lines to your `Dockerfile`: @@ -274,15 +274,15 @@ datadog-agent run & /opt/datadog-agent/embedded/bin/process-agent --config=/etc/datadog-agent/datadog.yaml ``` -For more advanced options in the Docker image, reference the [Datadog Agent Docker files][19]. +For more advanced options in the Docker image, reference the [Datadog Agent Docker files][24]. ## Contributing -See the [contributing documentation][20] to learn how to open an issue or PR to the [Heroku-buildpack-datadog repository][21]. +See the [contributing documentation][25] to learn how to open an issue or PR to the [Heroku-buildpack-datadog repository][26]. ## History -Earlier versions of this project were forked from the [miketheman heroku-buildpack-datadog project][22]. It was largely rewritten for Datadog's Agent version 6. Changes and more information can be found in the [changelog][23]. +Earlier versions of this project were forked from the [miketheman heroku-buildpack-datadog project][27]. It was largely rewritten for Datadog's Agent version 6. Changes and more information can be found in the [changelog][28]. ## FAQs / Troubleshooting @@ -292,32 +292,33 @@ Make sure you have `DD_DYNO_HOST` set to `true` and that `HEROKU_APP_NAME` has a ### After upgrading the buildpack or the agent, the agent is reporting errors when starting up -After an upgrade of the buildpack or agent, you must clear your build cache and recompile your application's slug. Check [the upgrading and slug recompilation section](#upgrading-and-slug-recompilation) for details. +After an upgrade of the buildpack or agent, you must recompile your application's slug. Check [the upgrading and slug recompilation section](#upgrading-and-slug-recompilation) for details. [1]: https://devcenter.heroku.com/articles/buildpacks [2]: https://docs.datadoghq.com/libraries [3]: https://app.datadoghq.com/account/settings#api -[4]: https://github.com/DataDog/heroku-buildpack-datadog/releases -[5]: https://docs.datadoghq.com/tagging/ -[6]: https://docs.datadoghq.com/dashboards/guide/how-to-graph-percentiles-in-datadog/ -[7]: https://docs.datadoghq.com/agent -[8]: https://devcenter.heroku.com/articles/dyno-metadata -[9]: https://devcenter.heroku.com/articles/log-runtime-metrics -[10]: https://docs.datadoghq.com/logs/guide/collect-heroku-logs -[11]: https://docs.datadoghq.com/logs/logs_to_metrics/ -[12]: https://docs.datadoghq.com/integrations/ -[13]: https://docs.datadoghq.com/integrations/redisdb/ -[14]: https://github.com/DataDog/integrations-core/blob/master/redisdb/datadog_checks/redisdb/data/conf.yaml.example -[15]: https://github.com/DataDog/integrations-extras/ -[16]: https://github.com/DataDog/integrations-extras/tree/master/ping -[17]: https://docs.datadoghq.com/agent/guide/agent-commands/#agent-status-and-information -[18]: https://devcenter.heroku.com/articles/slug-compiler -[19]: https://github.com/DataDog/datadog-agent/tree/master/Dockerfiles -[20]: https://github.com/DataDog/heroku-buildpack-datadog/blob/master/CONTRIBUTING.md -[21]: https://github.com/DataDog/heroku-buildpack-datadog -[22]: https://github.com/miketheman/heroku-buildpack-datadog -[23]: https://github.com/DataDog/heroku-buildpack-datadog/blob/master/CHANGELOG.md -[24]: https://devcenter.heroku.com/articles/using-multiple-buildpacks-for-an-app#viewing-buildpacks -[25]: https://github.com/heroku/heroku-buildpack-apt -[26]: https://github.com/jontewks/puppeteer-heroku-buildpack -[27]: https://github.com/lstoll/heroku-buildpack-monorepo +[4]: https://devcenter.heroku.com/articles/using-multiple-buildpacks-for-an-app#viewing-buildpacks +[5]: https://github.com/heroku/heroku-buildpack-apt +[6]: https://github.com/jontewks/puppeteer-heroku-buildpack +[7]: https://github.com/lstoll/heroku-buildpack-monorepo +[8]: https://github.com/DataDog/heroku-buildpack-datadog/releases +[9]: https://app.datadoghq.com/account/settings#api +[10]: https://docs.datadoghq.com/tagging/ +[11]: https://docs.datadoghq.com/dashboards/guide/how-to-graph-percentiles-in-datadog/ +[12]: https://docs.datadoghq.com/agent +[13]: https://devcenter.heroku.com/articles/dyno-metadata +[14]: https://devcenter.heroku.com/articles/log-runtime-metrics +[15]: https://docs.datadoghq.com/logs/guide/collect-heroku-logs +[16]: https://docs.datadoghq.com/logs/logs_to_metrics/ +[17]: https://docs.datadoghq.com/integrations/ +[18]: https://docs.datadoghq.com/integrations/redisdb/ +[19]: https://github.com/DataDog/integrations-core/blob/master/redisdb/datadog_checks/redisdb/data/conf.yaml.example +[20]: https://github.com/DataDog/integrations-extras/ +[21]: https://github.com/DataDog/integrations-extras/tree/master/ping +[22]: https://docs.datadoghq.com/agent/guide/agent-commands/#agent-status-and-information +[23]: https://devcenter.heroku.com/articles/slug-compiler +[24]: https://github.com/DataDog/datadog-agent/tree/master/Dockerfiles +[25]: https://github.com/DataDog/heroku-buildpack-datadog/blob/master/CONTRIBUTING.md +[26]: https://github.com/DataDog/heroku-buildpack-datadog +[27]: https://github.com/miketheman/heroku-buildpack-datadog +[28]: https://github.com/DataDog/heroku-buildpack-datadog/blob/master/CHANGELOG.md From 963b1543bf949b4aca3699a2d0a278b9456a2ee0 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 3 Mar 2021 12:29:37 +0100 Subject: [PATCH 058/175] Set 6.26 adn 7.26 as the pinned versions for the agent --- bin/compile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/compile b/bin/compile index b3df962..0ea6939 100644 --- a/bin/compile +++ b/bin/compile @@ -11,8 +11,8 @@ set -o pipefail # set -x # Set agent pinned version -DD_AGENT_PINNED_VERSION_6="6.25.0-1" -DD_AGENT_PINNED_VERSION_7="7.25.0-1" +DD_AGENT_PINNED_VERSION_6="6.26.0-1" +DD_AGENT_PINNED_VERSION_7="7.26.0-1" # Parse and derive params BUILD_DIR=$1 From 35c3cdf27a9c8c5a96e17b101655324705a10c18 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 3 Mar 2021 15:32:16 +0100 Subject: [PATCH 059/175] Prepare release 1.24 --- CHANGELOG.md | 6 ++++++ extra/datadog.sh | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77543cf..7fea46c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased & outstanding issues] - Non-https repo url and apt fetching +## [1.24] - 2021-03-03 + +### Added +- Datadog agent pinned versions are now `6.26.0` and `7.26.0` +- Datadog configuration path can now be configured with the `DD_HEROKU_CONF_FOLDER` environment variable + ## [1.23] - 2021-01-20 ### Added diff --git a/extra/datadog.sh b/extra/datadog.sh index e9379e1..6731557 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -50,7 +50,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="dev" +BUILDPACKVERSION="1.24" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # Include install method From a21ac95043f3a314b9546834568eb6452f67bf01 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 3 Mar 2021 15:52:34 +0100 Subject: [PATCH 060/175] Revert buildpack version back to dev --- extra/datadog.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index 6731557..e9379e1 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -50,7 +50,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="1.24" +BUILDPACKVERSION="dev" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # Include install method From 5e2e5ff5bad9bbf197c25e64b38369ef024fa25d Mon Sep 17 00:00:00 2001 From: Ruth Naebeck Date: Mon, 8 Mar 2021 18:13:39 -0500 Subject: [PATCH 061/175] Lint README for Documentation style --- README.md | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index f93d871..5ad00c4 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Once complete, the Datadog Agent is started automatically when each dyno starts. The Datadog Agent provides a listening port on `8125` for statsd/dogstatsd metrics and events. Traces are collected on port `8126`. ### Order of buildpacks -As explained in [Heroku's documentation][4], the last buildpack in the list will be used to determine the process type for the application. +As explained in [Heroku's documentation][4], the last buildpack in the list is used to determine the process type for the application. Buildpacks that install apt packages (e.g. [heroku-buildpack-apt][5], [puppeteer-heroku-buildpack][6]) or buildpacks that modify the `/app` folder (e.g. [heroku-buildpack-monorepo][7]) need to be added **before** the Datadog buildpack. For example, if your application uses the `ruby`, `datadog` and `apt` buildpacks, this would be a correct `heroku buildpacks` output: @@ -43,7 +43,7 @@ Buildpacks that install apt packages (e.g. [heroku-buildpack-apt][5], [puppeteer 3. heroku/ruby ``` -## Pinning a specific buildpack version and a specific Datadog agent version +## Pinning a specific buildpack version and a specific Datadog Agent version Heroku recommends to always use the latest commit of a buildpack. If you need to pin the buildpack version, you can do so by specifying the buildpack release tag: @@ -95,10 +95,10 @@ In addition to the environment variables shown above, there are a number of othe | `DD_PROCESS_AGENT` | *Optional.* The Datadog Process Agent is disabled by default. Set this to `true` to enable the Process Agent. Changing this option requires recompilation of the slug. Check [the upgrading and slug recompilation section](#upgrading-and-slug-recompilation) for details. | | `DD_SITE` | *Optional.* If you use the app.datadoghq.eu service, set this to `datadoghq.eu`. Defaults to `datadoghq.com`. | | `DD_AGENT_VERSION` | *Optional.* By default, the buildpack installs the latest 6.x version of the Datadog Agent available in the package repository. Use this variable to install older versions of the Datadog Agent (note that not all versions of the Agent may be available). This option takes precedence over `DD_AGENT_MAJOR_VERSION`. Changing this option requires recompiling the slug. Check [the upgrading and slug recompilation section](#upgrading-and-slug-recompilation) for details. | -| `DD_AGENT_MAJOR_VERSION` | *Optional.* By default, the buildpack installs the latest 6.x version of the Datadog Agent available in the package repository. Set this variable to `7` to install the latest 7.x version of the Datadog Agent. Check the [Python versions section](#python-and-agent-versions) for more information on the relation of the agent version and Python version. Changing this option requires recompiling the slug. Check [the upgrading and slug recompilation section](#upgrading-and-slug-recompilation) for details. | +| `DD_AGENT_MAJOR_VERSION` | *Optional.* By default, the buildpack installs the latest 6.x version of the Datadog Agent available in the package repository. Set this variable to `7` to install the latest 7.x version of the Datadog Agent. Check the [Python versions section](#python-and-agent-versions) for more information on the relation of the Agent version and Python version. Changing this option requires recompiling the slug. Check [the upgrading and slug recompilation section](#upgrading-and-slug-recompilation) for details. | | `DD_DISABLE_HOST_METRICS` | *Optional.* By default, the buildpack reports system metrics for the host machine running the dyno. Set this to `true` to disable system metrics collection. See the [system metrics section](#system-metrics) below for more information. | -| `DD_PYTHON_VERSION` | *Optional.* Starting with version `6.14.0`, Datadog Agent ships with Python versions `2` and `3`. The buildpack will only keep one of the versions. Set this to `2` or `3` to select the Python version you want the agent to keep. If not set, the buildpack will keep `2`. Check the [Python versions section](#python-and-agent-versions) for more information. Changing this option requires recompiling the slug. Check [the upgrading and slug recompilation section](#upgrading-and-slug-recompilation) for details. | -| `DD_HEROKU_CONF_FOLDER` | *Optional.* By default, the buildpack will look in the root of your application for a folder `/datadog` for any configuration files you wish to include, eg see [prerun.sh script](#prerun-script). This location can be overridden by setting this to your desired path. | +| `DD_PYTHON_VERSION` | *Optional.* Starting with version `6.14.0`, Datadog Agent ships with Python versions `2` and `3`. The buildpack only keeps one of the versions. Set this to `2` or `3` to select the Python version you want the Agent to keep. If not set, the buildpack keeps `2`. Check the [Python versions section](#python-and-agent-versions) for more information. Changing this option requires recompiling the slug. Check [the upgrading and slug recompilation section](#upgrading-and-slug-recompilation) for details. | +| `DD_HEROKU_CONF_FOLDER` | *Optional.* By default, the buildpack looks in the root of your application for a folder `/datadog` for any configuration files you wish to include, eg see [prerun.sh script](#prerun-script). This location can be overridden by setting this to your desired path. | For additional documentation, refer to the [Datadog Agent documentation][12]. @@ -106,9 +106,9 @@ For additional documentation, refer to the [Datadog Agent documentation][12]. Heroku dynos are ephemeral—they can move to different host machines whenever new code is deployed, configuration changes are made, or resouce needs/availability changes. This makes Heroku flexible and responsive, but can potentially lead to a high number of reported hosts in Datadog. Datadog bills on a per-host basis, and the buildpack default is to report actual hosts, which can lead to higher than expected costs. -Depending on your use case, you may want to set your hostname so that hosts are aggregated and report a lower number. To do this, Set `DD_DYNO_HOST` to `true`. This will cause the Agent to report the hostname as the app and dyno name (e.g. `appname.web.1` or `appname.run.1234`) and your host count will closely match your dyno usage. One drawback is that you may see some metrics continuity errors whenever a dyno is cycled. +Depending on your use case, you may want to set your hostname so that hosts are aggregated and report a lower number. To do this, Set `DD_DYNO_HOST` to `true`. This causes the Agent to report the hostname as the app and dyno name (for example `appname.web.1` or `appname.run.1234`) and your host count will closely match your dyno usage. One drawback is that you may see some metrics continuity errors whenever a dyno is cycled. -For this to work correctly, `HEROKU_APP_NAME` needs to be set. The easiest way to do this is by [enabling dyno metadata][13] Take into account that dyno metadata is not yet available in Private Spaces, in which case you will need to set `HEROKU_APP_NAME` manually. +For this to work correctly, `HEROKU_APP_NAME` needs to be set. The easiest way to do this is by [enabling dyno metadata][13]. **Note**: Dyno metadata is not yet available in Private Spaces, in which case you need to set `HEROKU_APP_NAME` manually. ## System metrics @@ -150,7 +150,7 @@ instances: **Note**: See the sample [redisdb.d/conf.yaml][19] for all available configuration options. -If the integration you are enabling is part of the [Community Integrations][20], you will also need to install the package as part of the [prerun script](#prerun-script). +If the integration you are enabling is part of the [Community Integrations][20], install the package as part of the [prerun script](#prerun-script). ``` agent-wrapper integration install -t datadog-== @@ -164,7 +164,7 @@ agent-wrapper integration install -t datadog-ping==1.0.0 ## Prerun script -In addition to all of the configurations above, you can include a prerun script, `/datadog/prerun.sh`, in your application. The prerun script will run after all of the standard configuration actions and immediately before starting the Datadog Agent. This allows you to modify the environment variables (for example, DD_TAGS or DD_VERSION), perform additional configurations, install community integrations, or even disable the Datadog Agent programmatically. +In addition to all of the configurations above, you can include a prerun script, `/datadog/prerun.sh`, in your application. The prerun script runs after all of the standard configuration actions and immediately before starting the Datadog Agent. This allows you to modify the environment variables (for example, DD_TAGS or DD_VERSION), perform additional configurations, install community integrations, or even disable the Datadog Agent programmatically. The example below demonstrates a few of the things you can do in the `prerun.sh` script: @@ -225,7 +225,7 @@ agent-wrapper status ## Python and Agent versions -Prior to version `6.14` the Datadog v6 agent shipped with Python version `2` embedded. Starting with `6.14`, and in preparation for Python version `2` End Of Life, announced for January 2020, the Datadog v6 agent ships with both Python versions `2` and `3`, to give customers enough time to migrate their custom checks to Python version `3`. The Heroku buildpack will only keep one of the versions. Set `DD_PYTHON_VERSION` to `2` or `3` to select the Python version you want the agent to keep. If not set, the buildpack will keep Python version `2`. If you are using custom checks that only work with Python version `2`, we recommend to migrate them to version `3` before its EOL. +Prior to version `6.14` the Datadog v6 Agent shipped with Python version `2` embedded. Starting with `6.14`, and in preparation for Python version `2` End Of Life, announced for January 2020, the Datadog v6 Agent ships with both Python versions `2` and `3`, to give customers enough time to migrate their custom checks to Python version `3`. The Heroku buildpack only keeps one of the versions. Set `DD_PYTHON_VERSION` to `2` or `3` to select the Python version you want the Agent to keep. If not set, the buildpack keeps Python version `2`. If you are using custom checks that only work with Python version `2`, we recommend to migrate them to version `3` before its EOL. Agent v7 only ships with Python version `3`. If you are not using custom checks or your custom checks are already migrated to version `3`, we recommend moving to Agent 7 as soon as possible. Starting with `6.15`, v7 releases with the same minor version share the same feature set, making it safe to move between those two. For example, if you are running `6.16` and you don't need Python version `2`, it is safe to jump to `7.16`. @@ -235,9 +235,9 @@ The Datadog buildpack does not collect logs from the Heroku platform. To set up ## Using Heroku with Docker images -This buildpack only works for Heroku deployments that use [Heroku's Slug Compiler][23]. If you are deploying your application in Heroku using Docker containers you will need to add the Datadog agent as part of your Docker image and start the agent as a different process in your container. +This buildpack only works for Heroku deployments that use [Heroku's Slug Compiler][23]. If you are deploying your application in Heroku using Docker containers, add the Datadog Agent as part of your Docker image and start the Agent as a different process in your container. -As an example, if you are building your Docker image using a Debian based OS, you will need to add the following lines to your `Dockerfile`: +As an example, if you are building your Docker image using a Debian based OS, add the following lines to your `Dockerfile`: ``` # Install GPG dependencies @@ -249,7 +249,7 @@ RUN sh -c "echo 'deb https://apt.datadoghq.com/ stable 7' > /etc/apt/sources.lis RUN apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 A2923DFF56EDA6E76E55E492D3A80E30382E94DE RUN apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 D75CEA17048B9ACBF186794B32637D44F14F620E -# Install the Datadog agent +# Install the Datadog Agent RUN apt-get update && apt-get -y --force-yes install --reinstall datadog-agent # Copy entrypoint @@ -264,7 +264,7 @@ COPY datadog-config/ /etc/datadog-agent/ CMD ["/entrypoint.sh"] ``` -In your Docker container entry point you will to start the Datadog Agent, the Datadog APM agent and the Datadog process agent: +In your Docker container entry point, start the Datadog Agent, Datadog APM Agent, and Datadog Process Agent: ``` #!/bin/bash @@ -284,15 +284,15 @@ See the [contributing documentation][25] to learn how to open an issue or PR to Earlier versions of this project were forked from the [miketheman heroku-buildpack-datadog project][27]. It was largely rewritten for Datadog's Agent version 6. Changes and more information can be found in the [changelog][28]. -## FAQs / Troubleshooting +## Troubleshooting ### Datadog is reporting a higher number of agents than dynos Make sure you have `DD_DYNO_HOST` set to `true` and that `HEROKU_APP_NAME` has a value set for every Heroku application. See the [Hostname section](#hostname) for details. -### After upgrading the buildpack or the agent, the agent is reporting errors when starting up +### After upgrading the buildpack or the Agent, the Agent is reporting errors when starting up -After an upgrade of the buildpack or agent, you must recompile your application's slug. Check [the upgrading and slug recompilation section](#upgrading-and-slug-recompilation) for details. +After an upgrade of the buildpack or Agent, you must recompile your application's slug. Check [the upgrading and slug recompilation section](#upgrading-and-slug-recompilation) for details. [1]: https://devcenter.heroku.com/articles/buildpacks [2]: https://docs.datadoghq.com/libraries From 5980e28988d8fb648d8bc8e6be9e9b30ba5eac1e Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Mon, 19 Apr 2021 10:58:32 +0200 Subject: [PATCH 062/175] Check signing keys for heroku-18 onwards --- bin/compile | 18 ++++++++++++------ etc/datadog.list | 2 +- etc/datadog7.list | 2 +- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/bin/compile b/bin/compile index 0ea6939..48b01c0 100644 --- a/bin/compile +++ b/bin/compile @@ -84,19 +84,25 @@ else mkdir -p "$APT_STATE_DIR/lists/partial" fi -# Install dependencies -topic "Updating apt caches for dependencies" -apt-get $APT_OPTIONS update | indent - - # Install GPG key topic "Install gpg key for Datadog APT Repository" APT_KEYRING="$CACHE_DIR/apt/trusted.gpg" +APT_KEYBOX="$CACHE_DIR/apt/temp.gpg" GPG_KEY_FILE="$BUILDPACK_DIR/etc/datadog.gpg" GPG_HOME_DIR="$BUILD_DIR/.gnupg" mkdir -p "$GPG_HOME_DIR" -gpg --ignore-time-conflict --no-options --no-default-keyring --homedir "$GPG_HOME_DIR" --keyring "$APT_KEYRING" --import "$GPG_KEY_FILE" | indent +# Remove old APT keyrings to avoid type conflicts +if [ -f $APT_KEYRING ]; then + rm $APT_KEYRING +fi + +if [[ $STACK == "heroku-16" ]]; then + gpg --ignore-time-conflict --no-options --no-default-keyring --homedir "$GPG_HOME_DIR" --keyring "$APT_KEYRING" --import "$GPG_KEY_FILE" | indent +else + gpg --ignore-time-conflict --no-options --no-default-keyring --homedir "$GPG_HOME_DIR" --keyring "$APT_KEYBOX" --trustdb "$APT_KEYRING" --import "$GPG_KEY_FILE" | indent + gpg --no-default-keyring --keyring "$APT_KEYBOX" --export > $APT_KEYRING +fi # Prior to Agent 6.14 there was only 1 python version DD_AGENT_BASE_VERSION="6.14" diff --git a/etc/datadog.list b/etc/datadog.list index 40bb38b..d1bc010 100644 --- a/etc/datadog.list +++ b/etc/datadog.list @@ -1,3 +1,3 @@ # We're using insecure http here. Eventually we should fix # the apt-transport-https issue. -deb [trusted=yes] http://apt.datadoghq.com/ stable 6 +deb http://apt.datadoghq.com/ stable 6 diff --git a/etc/datadog7.list b/etc/datadog7.list index 809d13d..e40998f 100644 --- a/etc/datadog7.list +++ b/etc/datadog7.list @@ -1,3 +1,3 @@ # We're using insecure http here. Eventually we should fix # the apt-transport-https issue. -deb [trusted=yes] http://apt.datadoghq.com/ stable 7 +deb http://apt.datadoghq.com/ stable 7 From e4a729849fa25dbda5f780163b3c8937624839ed Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Mon, 19 Apr 2021 13:26:30 +0200 Subject: [PATCH 063/175] pin default versions to 6.27 and 7.27 --- bin/compile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/compile b/bin/compile index 0ea6939..a01013d 100644 --- a/bin/compile +++ b/bin/compile @@ -11,8 +11,8 @@ set -o pipefail # set -x # Set agent pinned version -DD_AGENT_PINNED_VERSION_6="6.26.0-1" -DD_AGENT_PINNED_VERSION_7="7.26.0-1" +DD_AGENT_PINNED_VERSION_6="6.27.0-1" +DD_AGENT_PINNED_VERSION_7="7.27.0-1" # Parse and derive params BUILD_DIR=$1 From f9296ac896967077957c4d1c6b55fe7be85f590b Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Mon, 19 Apr 2021 16:22:17 +0200 Subject: [PATCH 064/175] Move repos to HTTPS --- etc/datadog.list | 4 +--- etc/datadog7.list | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/etc/datadog.list b/etc/datadog.list index d1bc010..03751bc 100644 --- a/etc/datadog.list +++ b/etc/datadog.list @@ -1,3 +1 @@ -# We're using insecure http here. Eventually we should fix -# the apt-transport-https issue. -deb http://apt.datadoghq.com/ stable 6 +deb https://apt.datadoghq.com/ stable 6 diff --git a/etc/datadog7.list b/etc/datadog7.list index e40998f..e7c5946 100644 --- a/etc/datadog7.list +++ b/etc/datadog7.list @@ -1,3 +1 @@ -# We're using insecure http here. Eventually we should fix -# the apt-transport-https issue. -deb http://apt.datadoghq.com/ stable 7 +deb https://apt.datadoghq.com/ stable 7 From 7106a302e0b14ae7fbd4964f33ea6eacda598c60 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Mon, 19 Apr 2021 16:45:26 +0200 Subject: [PATCH 065/175] Prepare release 1.25 --- CHANGELOG.md | 9 +++++++-- extra/datadog.sh | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7fea46c..8a64267 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). -## [Unreleased & outstanding issues] -- Non-https repo url and apt fetching +## [1.25] - 2021-04-19 + +### Added +- Datadog agent pinned versions are now `6.27.0` and `7.27.0` + +### Fixed +- HTTPS repo fetching ## [1.24] - 2021-03-03 diff --git a/extra/datadog.sh b/extra/datadog.sh index e9379e1..eb31f85 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -50,7 +50,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="dev" +BUILDPACKVERSION="1.25" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # Include install method From 3c3618871d5b23b0ec116a1851971f0f68f5d2d4 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Mon, 19 Apr 2021 17:25:57 +0200 Subject: [PATCH 066/175] Revert version back to dev --- extra/datadog.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index eb31f85..e9379e1 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -50,7 +50,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="1.25" +BUILDPACKVERSION="dev" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # Include install method From 0c933bd71b7eb147c6307df24585ae41e6c7710e Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Tue, 20 Apr 2021 10:29:57 +0200 Subject: [PATCH 067/175] Fix instructions for the user to select the app --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5ad00c4..fe7b8d6 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ heroku create heroku buildpacks:add heroku/ruby # Enable Heroku Labs Dyno Metadata -heroku labs:enable runtime-dyno-metadata -a $(heroku apps:info|grep ===|cut -d' ' -f2) +heroku labs:enable runtime-dyno-metadata -a # Add this buildpack and set your Datadog API key heroku buildpacks:add --index 1 https://github.com/DataDog/heroku-buildpack-datadog.git @@ -71,7 +71,7 @@ To upgrade this buildpack and/or to change any of these options, for example `DD ```shell # Set new version of the Agent -heroku config:set DD_AGENT_VERSION= -a appname +heroku config:set DD_AGENT_VERSION= -a # Rebuild your slug with the new Agent version: git commit --allow-empty -m "Rebuild slug" From e82999b76a044a2a4ba6eeba8d93c25a45874eac Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Tue, 20 Apr 2021 10:45:53 +0200 Subject: [PATCH 068/175] Force a rebuild on setup --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index fe7b8d6..e7e157a 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,8 @@ heroku labs:enable runtime-dyno-metadata -a heroku buildpacks:add --index 1 https://github.com/DataDog/heroku-buildpack-datadog.git heroku config:add DD_API_KEY= -# Deploy to Heroku +# Deploy to Heroku forcing a rebuild +git commit --allow-empty -m "Rebuild slug" git push heroku master ``` From 32a9d4cbb5a2100e5dce20e2275d9c276967bd5a Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Tue, 20 Apr 2021 16:48:38 +0200 Subject: [PATCH 069/175] Improve getting started section --- README.md | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index e7e157a..b7d7958 100644 --- a/README.md +++ b/README.md @@ -4,31 +4,40 @@ This [Heroku buildpack][1] installs the Datadog Agent in your Heroku dyno to col ## Installation -To add this buildpack to your project, as well as set the required environment variables: +This guide assumes that you already have your application running on Heroku. Please, refer to Heroku documentation to learn how to deploy your application to Heroku. + +1. Go to [Datadog API settings][3] and copy your Datadog API key. Export it to an environment variable: + + ```shell + export DD_API_KEY= + ``` + +2. Export your application name to the APPNAME environment variable: ```shell -cd +export APPNAME= +``` -# If this is a new Heroku project -heroku create +3. Add the Datadog buildpack to your project: -# Add the appropriate language-specific buildpack. For example: -heroku buildpacks:add heroku/ruby +```shell +cd -# Enable Heroku Labs Dyno Metadata -heroku labs:enable runtime-dyno-metadata -a +# Enable Heroku Labs Dyno Metadata to set HEROKU_APP_NAME env variable automatically +heroku labs:enable runtime-dyno-metadata -a $APPNAME + +# Set hostname in Datadog as appname.dynotype.dynonumber for metrics continuity +heroku config:add DD_DYNO_HOST=true # Add this buildpack and set your Datadog API key heroku buildpacks:add --index 1 https://github.com/DataDog/heroku-buildpack-datadog.git -heroku config:add DD_API_KEY= +heroku config:add DD_API_KEY=$DD_API_KEY # Deploy to Heroku forcing a rebuild git commit --allow-empty -m "Rebuild slug" git push heroku master ``` -Replace `` with your [Datadog API key][3]. - Once complete, the Datadog Agent is started automatically when each dyno starts. The Datadog Agent provides a listening port on `8125` for statsd/dogstatsd metrics and events. Traces are collected on port `8126`. From af5a3ea9a54743f136c65478e288f291df762b68 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Tue, 4 May 2021 12:45:41 +0200 Subject: [PATCH 070/175] Clean the APT state dir between versions and stacks --- bin/compile | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/compile b/bin/compile index fb44ea6..b6e0c02 100644 --- a/bin/compile +++ b/bin/compile @@ -80,6 +80,7 @@ else # STACK or DD_AGENT_VERSION changed, clean up APT cache topic "Detected Stack and/or agent version changes, flushing cache" rm -rf $APT_CACHE_DIR + rm -rf $APT_STATE_DIR mkdir -p "$APT_CACHE_DIR/archives/partial" mkdir -p "$APT_STATE_DIR/lists/partial" fi From f5e40ad26fb26689f7009e2c9ac208d2bbb32c6c Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Thu, 13 May 2021 10:54:41 +0200 Subject: [PATCH 071/175] Add agent status instructions to troubleshoot section --- README.md | 110 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/README.md b/README.md index b7d7958..b1cd39f 100644 --- a/README.md +++ b/README.md @@ -296,6 +296,116 @@ Earlier versions of this project were forked from the [miketheman heroku-buildpa ## Troubleshooting +### Getting the agent status + +If you have set up the buildpack and you are not getting some of the data you expected in Datadog, you can run the status command for the Datadog agent to help you finding the cause. + +```shell +# Export the name of your Heroku application as an environment variable +export APPNAME=your-application-name + +heroku ps:exec -a $APPNAME + +# Establishing credentials... done +# Connecting to web.1 on ⬢ ruby-heroku-datadog... +# DD_API_KEY environment variable not set. Run: heroku config:add DD_API_KEY= +# The Datadog Agent has been disabled. Unset the DISABLE_DATADOG_AGENT or set missing environment variables. + +~ $ +``` + +You can ignore the warnings about the DD_API_KEY not being set. This is normal. The reason is that [Heroku doesn’t set configuration variables for the SSH session itself](https://devcenter.heroku.com/articles/exec#environment-variables), but the Datadog agent process was able to access those. + +Once inside the SSH session, execute the Datadog status command. + +```shell +~ $ agent-wrapper status + +Getting the status from the agent. + +=============== +Agent (v7.27.0) +=============== + +[...] + +``` + +We will hightligth some of the sections you should focus on, depending on what you are trying to debug. + +#### I am not getting any data in Datadog + +Make sure that the `status` command runs correctly and that this section of the output tells you that your API key is valid: + +``` + API Keys status + =============== + API key ending with 68306: API Key valid +``` + +#### Check integrations + +To check if the integration you have enabled is running correctly, focus on the `Collector` section and verify that your check is running correctly: + +``` +========= +Collector +========= + + Running Checks + ============== + +[...] + postgres (5.4.0) + ---------------- + Instance ID: postgres:e07ef94b907fe733 [OK] + Configuration Source: file:/app/.apt/etc/datadog-agent/conf.d/postgres.d/conf.yaml + Total Runs: 4,282 + Metric Samples: Last Run: 15, Total: 64,230 + Events: Last Run: 0, Total: 0 + Service Checks: Last Run: 1, Total: 4,282 + Average Execution Time : 43ms + Last Execution Date : 2021-05-13 08:15:46 UTC (1620893746000) + Last Successful Execution Date : 2021-05-13 08:15:46 UTC (1620893746000) + metadata: + version.major: 13 + version.minor: 2 + version.patch: 0 + version.raw: 13.2 (Ubuntu 13.2-1.pgdg20.04+1) + version.scheme: semver +``` + +#### Check APM agent + +If you have instrumented your application for APM and not getting traces in Datadog, you can check that the APM agent is running correctly and collecting traces: + +```` +[...] +========= +APM Agent +========= + Status: Running + Pid: 63 + Uptime: 64702 seconds + Mem alloc: 10,331,128 bytes + Hostname: ruby-heroku-datadog.web.1 + Receiver: localhost:8126 + Endpoints: + https://trace.agent.datadoghq.com + + Receiver (previous minute) + ========================== + From ruby 2.6.6 (ruby-x86_64-linux), client 0.48.0 + Traces received: 11 (14,181 bytes) + Spans received: 33 + + Default priority sampling rate: 100.0% + Priority sampling rate for 'service:ruby-heroku-datadog,env:': 100.0% + Priority sampling rate for 'service:ruby-heroku-datadog,env:development': 100.0% + +[...] +``` + ### Datadog is reporting a higher number of agents than dynos Make sure you have `DD_DYNO_HOST` set to `true` and that `HEROKU_APP_NAME` has a value set for every Heroku application. See the [Hostname section](#hostname) for details. From c3f498b41fb0855c36eaf8433b977eead31aa143 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Mon, 17 May 2021 11:25:27 +0200 Subject: [PATCH 072/175] Set the dyno id as a host alias --- extra/datadog.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/extra/datadog.sh b/extra/datadog.sh index e9379e1..bc58fd2 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -53,6 +53,9 @@ DYNOTYPE=${DYNO%%.*} BUILDPACKVERSION="dev" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" +# We want always to have the Dyno ID as a host alias to improve correlation +export DD_HOST_ALIASES="$DYNOHOST" + # Include install method echo -e "install_method:\n tool: heroku\n tool_version: heroku\n installer_version: heroku-$BUILDPACKVERSION" > "$DD_INSTALL_INFO" From 99c3ec59bd85330970ac05e7c377591677b95a3d Mon Sep 17 00:00:00 2001 From: Slavek Kabrda Date: Tue, 18 May 2021 15:15:49 +0200 Subject: [PATCH 073/175] Implement improvements in APT keys management * Get keys from keys.datadoghq.com * Add 'signed-by' option to all sources list lines, point them to the dynamically constructed keyring --- README.md | 16 +++-- bin/compile | 27 ++++---- etc/datadog.gpg | 127 ----------------------------------- etc/datadog.list | 2 +- etc/datadog7.list | 2 +- test/compile_and_run_test.sh | 4 ++ 6 files changed, 30 insertions(+), 148 deletions(-) delete mode 100644 etc/datadog.gpg diff --git a/README.md b/README.md index b7d7958..2d9b9f0 100644 --- a/README.md +++ b/README.md @@ -252,12 +252,20 @@ As an example, if you are building your Docker image using a Debian based OS, ad ``` # Install GPG dependencies RUN apt-get update \ - && apt-get install -y gpg apt-transport-https gpg-agent curl ca-certificates + && apt-get install -y gnupg apt-transport-https gpg-agent curl ca-certificates # Add Datadog repository and signing keys -RUN sh -c "echo 'deb https://apt.datadoghq.com/ stable 7' > /etc/apt/sources.list.d/datadog.list" -RUN apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 A2923DFF56EDA6E76E55E492D3A80E30382E94DE -RUN apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 D75CEA17048B9ACBF186794B32637D44F14F620E +ENV DATADOG_APT_KEYRING="/usr/share/keyrings/datadog-archive-keyring.gpg" +ENV DATADOG_APT_KEYS_URL="https://keys.datadoghq.com" +RUN sh -c "echo 'deb [signed-by=${DATADOG_APT_KEYRING}] https://apt.datadoghq.com/ stable 7' > /etc/apt/sources.list.d/datadog.list" +RUN touch ${DATADOG_APT_KEYRING} +RUN curl -o /tmp/DATADOG_APT_KEY_CURRENT.public "${DATADOG_APT_KEYS_URL}/DATADOG_APT_KEY_CURRENT.public" && \ + gpg --ignore-time-conflict --no-default-keyring --keyring ${DATADOG_APT_KEYRING} --import /tmp/DATADOG_APT_KEY_CURRENT.public +RUN curl -o /tmp/DATADOG_APT_KEY_F14F620E.public "${DATADOG_APT_KEYS_URL}/DATADOG_APT_KEY_F14F620E.public" && \ + gpg --ignore-time-conflict --no-default-keyring --keyring ${DATADOG_APT_KEYRING} --import /tmp/DATADOG_APT_KEY_F14F620E.public +RUN curl -o /tmp/DATADOG_APT_KEY_382E94DE.public "${DATADOG_APT_KEYS_URL}/DATADOG_APT_KEY_382E94DE.public" && \ + gpg --ignore-time-conflict --no-default-keyring --keyring ${DATADOG_APT_KEYRING} --import /tmp/DATADOG_APT_KEY_382E94DE.public + # Install the Datadog Agent RUN apt-get update && apt-get -y --force-yes install --reinstall datadog-agent diff --git a/bin/compile b/bin/compile index fb44ea6..350d67e 100644 --- a/bin/compile +++ b/bin/compile @@ -86,23 +86,17 @@ fi # Install GPG key topic "Install gpg key for Datadog APT Repository" -APT_KEYRING="$CACHE_DIR/apt/trusted.gpg" -APT_KEYBOX="$CACHE_DIR/apt/temp.gpg" -GPG_KEY_FILE="$BUILDPACK_DIR/etc/datadog.gpg" +DATADOG_APT_KEYRING="$BUILDPACK_DIR/datadog-archive-keyring.gpg" +DATADOG_APT_KEYS=("DATADOG_APT_KEY_CURRENT.public" "DATADOG_APT_KEY_F14F620E.public" "DATADOG_APT_KEY_382E94DE.public") +DATADOG_APT_KEYS_URL="https://keys.datadoghq.com" GPG_HOME_DIR="$BUILD_DIR/.gnupg" mkdir -p "$GPG_HOME_DIR" +touch $DATADOG_APT_KEYRING -# Remove old APT keyrings to avoid type conflicts -if [ -f $APT_KEYRING ]; then - rm $APT_KEYRING -fi - -if [[ $STACK == "heroku-16" ]]; then - gpg --ignore-time-conflict --no-options --no-default-keyring --homedir "$GPG_HOME_DIR" --keyring "$APT_KEYRING" --import "$GPG_KEY_FILE" | indent -else - gpg --ignore-time-conflict --no-options --no-default-keyring --homedir "$GPG_HOME_DIR" --keyring "$APT_KEYBOX" --trustdb "$APT_KEYRING" --import "$GPG_KEY_FILE" | indent - gpg --no-default-keyring --keyring "$APT_KEYBOX" --export > $APT_KEYRING -fi +for key in "${DATADOG_APT_KEYS[@]}"; do + curl -o "/tmp/${key}" "${DATADOG_APT_KEYS_URL}/${key}" | indent + gpg --ignore-time-conflict --no-options --no-default-keyring --homedir "$GPG_HOME_DIR" --keyring "$DATADOG_APT_KEYRING" --import "/tmp/${key}" | indent +done # Prior to Agent 6.14 there was only 1 python version DD_AGENT_BASE_VERSION="6.14" @@ -129,9 +123,12 @@ if [ "$DD_AGENT_VERSION" != "$(echo -e "$DD_AGENT_BASE_VERSION\n$DD_AGENT_VERSIO cp "$APT_REPO_FILE_7" "$APT_REPO_FILE" fi +# Replace the placeholder in the source file by the actual path to the constructed keyring +sed -i "s|SIGNED_BY_PLACEHOLDER|${DATADOG_APT_KEYRING}|" $APT_REPO_FILE + # Install Datadog Agent topic "Updating apt caches for Datadog Agent" -APT_OPTIONS="$APT_OPTIONS -o Dir::Etc::Trusted=$APT_KEYRING -o Dir::Etc::SourceList=$APT_REPO_FILE" +APT_OPTIONS="$APT_OPTIONS -o Dir::Etc::SourceList=$APT_REPO_FILE" apt-get $APT_OPTIONS update | indent # Accommodate for earlier pinned versions. Give deprecation warning. diff --git a/etc/datadog.gpg b/etc/datadog.gpg deleted file mode 100644 index c47b876..0000000 --- a/etc/datadog.gpg +++ /dev/null @@ -1,127 +0,0 @@ ------BEGIN PGP PUBLIC KEY BLOCK----- -Version: SKS 1.1.6 - -mQINBFd0KRUBEACr4s2I9/iXtRYQas0ux+OmQ6PUuPVxYT8nBTnqQV8HTlpigUUZVRXDKH6/ -XEl5gIU3vjh8i0GTZ+F4bB0pE+nwOjuM4F7SDE5JvYd75EfKvyqJs7PKTgVaSGLs6b8t95ma -jWw6Svmh/IbP7TiDe/hxdcucQ3VeF/ufdySuPsUI+EAamNy+IqBGRTlmxk1vGcW8vHh1TNZV -hoA98HQbqTqXqEq051r2AENX/kR/5xowhc0s2JN3zbXSXpSN5rulqHzJ41kaL7DVQJmRmvn9 -tT5qXelKtr/Hhw0PkUwZ1vJ5GzGkx7zGD/yk4w0R9MyMrX94F0Dgma/3DddiuWNfCCNhsN4Q -mD+zBx8UPD1Ym9ViUGcVERw6khkThNjUCqdCTwTnS7YRzyFjaxCBDRpzGnPa1auQgiioSl3R -+bRj893tyRqG00D/viyIeITTPK/JdDqnUVzDHv+tpzZsYEgaBLmopQkzjlGIS8WcS3Iv5dr2 -yI6tatjSWdqqgFvesvgV9AS0nrIFcIORDqXWrudTv0CEZrhOKCrilPxDHhcGR9ADVap/Hyu1 -naQYtGt3w2ssx8d4/5IXkicNkv4LJ1AQ1bRgFJpZsiMVVjQ0qMAQL/CltumyVfdNaZtk+fuh -xQmQ2bMwQ5mv5Y7yE0shI5nE8+Kh7FEyUCTPOmnjpjsDyNiClQARAQABtCREYXRhZG9nLCBJ -bmMgPHBhY2thZ2VAZGF0YWRvZ2hxLmNvbT6JAj0EEwECACcCGwMFCQtHNQACHgECF4AFAld0 -KTAFCwkIBwMFFQoJCAsFFgIDAQAACgkQ06gOMDgulN5oEg//XCqwqSWO2BDCDPqWwjucwrx6 -taLOh1f3YvKQaAWep7Bi5OR094KjrhSNdrtJQGcr9SWi/m729Ehn+ZJ19Dy82H/Cn81BCMLt -5fF8HuFcj1HmMUBZxYUGBqzYtw14A8D0QxOyYxte+aSTp0p8ySopi+awn5Pl0xM8lNx+9AGu -8EFVPGe7KXN9iY2RlTOZz0Vvez/QRs99kp5cCGNXpTDb3FUBdJMsIlNX4oX6Jpof8kBYmBh1 -O7N1wmn1k6mIunxGw+7WIR1EuR1s37ts6LxTtfgiglPKs4aEQ+2BP4JQisHUWkfVgnFmGIYL -qNsz+8eEIeLxXIMXwPZos9iME8BF1Axf+9TYg+87WrfbPpOm35SR3+F1aGNaUsMqLTnzGvHR -WKFh8grT9nIwkHn14gGd39p+4nS5t3znHNZbqQLwztNFgRwu554kOE6XLT5AW4bfhzgrTm/F -ZwvfLikaleG3F1ZBNIA9t/LirYLmFFBSkT8ro6+ffWTl/xixakSGRu7LZFD9YYlT1ChtMT9I -M8Ywu3UoYKwKJYhVo0/wHvdtvDlCUF2VffjueodqBBoBXekhzpmAJEJlFqsTvMhJFCqeZic7 -P5Uk5icuBqbCIuvgmpMrAQFkrf4mObH4lzltCRREFzNmJFfmtX/GLZXSvXaiTPThw+rPIBE3 -dhORObhtZIuJAj4EEwECACgFAld0KRUCGwMFCQtHNQAGCwkIBwMCBhUIAgkKCwQWAgMBAh4B -AheAAAoJENOoDjA4LpTeRfwP+QHmQzAWMrgv+0mUoVX1T13Uoj4S5d8r4JVhbzqc4W8RF75h -jIGCl5pDpNDUf4xJWhky2EQv1WtblPsW56IxsZenJtf1Fj8ubZQB5pumJbAF/DduEAQmP+VH -DnfRIj866HywqUyN+EtJmGZttz9s9GRqgVMuDKG4tW7PSLncc5tESGOnm1GTLE1n5JkukmPB -8aomnv7qOp2cdbTE56ynqfYoTxqg5x+7dJnHBm5Ih7i2QA6e0Q0/QBAESMYvybM9zb3ED762 -pfU11Fgvjj+ZXCQ1EvibIwr/DYqBzuDGwyoKSXqqJPxQ4PazpfW/W3CnAkqO+VbQ5+F7nojx -5H4ZCaMD8UCLgecj3qRl9+JjoYCmcRorrCG6Idaf7b1FfT7S07JZoR7nomrm9EzigK1TmDZb -dTc5ploUEZGi9cwKXWsBJl5KlW4JlqVHHyu52yobXuKlmuQecmQD/nTlCkRnZMmVhV7kfSjq -huMCwi+yAiEknOFmBLth8hsQ3dX6ZeUtc26lDalx7uaz8zHhA9QkLoBW2L7q1fMLSFNxt1Kv -dPjHkc7RbiviywOyZemfCMXR7911Duk8kPk411O1QvadKOEmPRn/Xq44ztJc6G+Zpl/QKBm3 -iOJzRxmupJvBLcTVzeYXQ2EiSEimJA4ZBxcGaiIFnVp7reD+tMKU8w+O8rIvuQINBFd0KWQB -EACsFZnTfSi7mJXb3pitRlI/w4U+dkwF3u3v4prKAuibY6DjC1gXhB1UoB4NflcHboMsT+Rl -x9WfMUwAoRtdrU/VUrrusnvqWqynwt05ejKkGLJDO414UJE7xYtxWjbjnsP3/c896k4G/7ka -TCL+AmHQWZGRcealSkuW7FpyNP2i7Vto2FT1hB/ckzsN+KdABtq7HX96DmOARucdmpG0y1k4 -PrGoxPsVbJjvXHi5AjigWga3BJHGXohsTJTYG4fbaMLynOL3QY98FYAygf7yVe0ZDpHddVU8 -o036vLbg/gKEhwLSaG95oYLUFekuvwb4HJFwdSR6A4k+Y/Pu0unf5h3cfJMX7Xml5A0FpjRH -PBSLVh/qxjzHzCl270HJSSkEw+TNvgi6cbkT6L+jR2V9eAgYdWmx0eHlCoBFKp9xLjDX0IIq -HDUv2mjyKipWpZMk7adFfFB4HAV+qTsjk/y0+S8XW1W1kstpAoddJaaNWQ4sTt0uD7J1Y69Y -hM0hY1cV5v/KQ+eIEB7T2DxoTgwCVbA+MiNBvzjVzJIHGjUPTf93PFYAaN+pfUjSv3OukjGo -loGVK6W+jRYIQUy+pGKaoGfRpAzZxRTjl5QrAJQR0y/5nK8ecOap4qCDnDv4IZ7vgtEuB+IO -VXHUSRsO8g7BJ5/ZxfjDyipKlG2PT2ABX9mFtQARAQABiQREBBgBAgAPBQJXdClkAhsCBQkL -RzUAAikJENOoDjA4LpTewV0gBBkBAgAGBQJXdClkAAoJECS+tDb0MvbgGcwP/21gDcfIs/vv -OBGWtSzej7Tyqg3bBVt7foNaKiOJ3YRwWYACsAt5qi129CyI9wHOF0OPebkT6rqK24+x/LkY -gZqzp1ZqedgHMUmGKJPDp+QSBWy2qfvc38lDthf0PlN1USupBY3rW0N5LjbmMHPfJBk1bmMN -2dAlO2HqV014hfU7fRvVa5G7HgH0qNvd5BmUxWUQ06KDj9uK2pvHPTZ92MEsrRitaz40yytY -eQN7tCD8kyP5f3DhYms0qfqR4h3pFV7Jwxkprl4ZJp3gq48zYf4lMcmpad/KntPyaGAnAhpm -D473dURNHqgNR0dfIy93RDtY8tdNFW9/7z81L/b8OkZbsSVKZePojMgzorMFqxh3IOrVC8ww -YKbwrEAMidNJgRVN7R2yKz9f1ne292MkhZPYCJO7XMMor0OBR5DEdPMSh9wAYnji+tjxjWzY -waEZOvnpaSVv3YIu6+SSmzjXe2xm7C8o0Ln+fOx67fK3BoBWtAW/J+DUS5Nm9FHL+JymZbP8 -93uBiKP5OXKXz5TU0er9KeEUeENw1/uwanfrr+GOuSN4BEDl6i5/e63+pm4TD4BOnB4dXpRt -JqpyCtLmn9NUKxjJfl4FgUwGuVKRZin6HeDVw/k84iCKN72NbCjgc+zX5cdE/OeS1m7qjJ5V -xMDLWXlRmKFGh9bfP/6TrKOapxgP/A/eaHfDUifv421SHnHCi8YsvGm/q+DPCJXGnW05NGAi -00ojCWPMi8sJCYkCMQx3jSpIneFdWlB5xe3eSId5Q8V+J9uhApSyT8d3kX/b0cHfLE/1eWoe -izZM8vuJj1fGlDwJpRy41uZ8wNiB614VHWUusphP0068JV4cSuVL6tHCE4A53Vux2Whb/L1R -Kdf5ukL9u/poDDTgEhCDLb8j4iADt4HvLsobtnYhS7eVJhSHCkcp26OVUMztx56gk9+N4O2+ -delBYhJqRju2sCMwtCdK9IWcXwB7ROISByXcw/Tg5Bc8PKYe2pdrnNedTze7wiJ7qWUjIN1M -QXOgqft8zVbXNEniRPyMU4xnhqhsDqcY6UGrKRoH+gpjeyhn0rE52fMC2S3svFsuyTYZOq2/ -X+MQez3mDPpt33LF5zmbBRa9Ng97RKpBLxi3I8vr5s19Pz8C9GbWDlB+Eq9h/5znyNwzhGx2 -U4fVYk/aQdn7G39qWBDtq3c0tkZXvP/CvGncL56ga1Hnv0vqoI4kTwzFeM5Xm9A/SitMBbdi -FZAfAQ0XtFeSYgVC/fmF4SJoh8G31noR/nWwO5c4F+5m6OFHZ/O/Uj2LmLiLENVHW0OtXlhI -dwZk+SZ/k6NsVvlelUVDwIWVEN0kDgVT8zHl45P/AzlZoaiyJubGlfJLGFEGMwo7uQINBFd0 -KgMBEACn/BFNyLL5OQ/ZGBj0glHdVCz9QSCrPTZIhrfNxjOgiZ7EMX6wc1yRUC/k4CNVYxea -ZRiCY382U9eTQEG1wc67P8WajrstYe9MD7ANEAPs+PGXMEhIR976V0o0fpjH8Cxd86Zugn2z -0wQWb/MT9TCrYpt/amlLb6rJThN5Xq5l0Px7L2CjEiHAUcz5+Kj2CwfXfUUi9xbCI5lhYltS -4gt7HrfXkO7D7uSP8eG76lsWiavD2HjIqmPhQN2MJiSJ8Z0bekqPOR6LLPdTUP9UoxlA+eMx -0bp+t8jlC7O5CBuSd2i8RIsx4Yc9AObLKD3KKrhJsVhY6picDK5gjmtPdEg3MVnqD4M75/F2 -zwyvt3Nc8JnDUxgGkKLnKSbFxZfq06/D0/iC5kCYTe+Q9DUAMxYzgyfwFbSBPGAes7RykVaC -ybUhj19bamOeOercdUZw/MD773Q/cWM0po31fVYWK5Boq+04+4OxNoVKzwDQB8XMLoYSw7um -kMEaqEJYcBBRAYdaM8SHB4JJCtpB3RzoPKc6jtjttWGHd6Hms0AYE8b/ICYkmA/1ldDFAaje -v+pB+nNxyzVWVatZ3FYsLzzSEcXLFaWvmRO4t7nSSofJOy25aaPltkVSzNnR9MgWPstnCXBa -YbkQH/cApPJ7TTatcSVCbqtbFXwV/2TG3MV7IK+2ewARAQABiQREBBgBAgAPBQJXdCoDAhsC -BQkLRzUAAikJENOoDjA4LpTewV0gBBkBAgAGBQJXdCoDAAoJEEtFkwGDh+6vLoIP/1xUxsS9 -AikK0iCabJV0O0hLj4nc396QXuc42XiljEzHxWkN3B90ShdcwEA2VSguCMsTOcoD/xOFpeNr -4kMWcAzLWzbxVeN2MCORA8W3WRnv8ABhYLbwWFIcWTZYiAihmHUjOqB8Z+fQP9jnZrXIpqOj -nXTp4JT4tiua2KbbEGG4mu3/Es5MfERfIYaMrL0q7pnM7zhZXrHLf+dcUmzdO+DZMjlfxxF2 -OWijegYSVopdUEaX8YqVbKiH1nhEPrIPifaB3uLBGBgUQQsnaOB6EHLuA2QM6get+7SarAQK -MDCuoxeIOhiYQMs+rB80m4WPR9iD5zbdAuh5m8suF7J9Flep3yeNau6cp82hRqFKcNLs2Psv -cB0GVA5mp5MvLAeS6zo8JSyQG0jf6JJpdg6PPwtm+NFkpCnDJy3vsxjxwhlXRAow8zr1Z6IO -gAVaIjnB12kDELgzS0K1721sq5t4FWeIVsO1Etiru6iAHm7KXEBcVQXD6/XjG0OPFHoa9TCx -KXbQcly6GuV6oeZLx1tSeccbGX69tOds/iWzj7jb2JhIsTm74U2ghnFgxr+sMpw/SRN0v+qn -Hvhm54g9G1cH5WdC4A7D8+VgpJYL3MllWK4EeyrGmXfJGPUbyaTHIrOgZSKM1/yu+DxsQaIB -hJgSNOzfuol7+Oom/gyJTlX/H+D+IlgQAJz7z6qtrO7//+NCtDB+EU7IE6U154kWzNbtx8Zv -AyJINh6NSHgXEmMhLzoHZdnB9tRF+APu6NEP/dOKrgZHqZNso65XVoYBNWCn2IO8zPRT8Qv6 -iZxfiksgNmbJYEBYwcxz1AZ1v1juL5iHb8EuCZ0O7fNVRtLkwucFFszPyaGPhWySKQT1oDt6 -S2uBv0CmPf7Jm36FNuh9mxREFxrSITO7/erYCNzqoIqswXwWJvBecOT9JmxUT65f6QjxM0Js -hmnar//ovu6jjfnuGIjLqlEdQIXLTq8qh20TsgtacHRo/nbEOWEfkq6Uc+qWL9dskD9DIks8 -lYoR6KetrMzdrkj3+mv2UpqrpT6HblluWQFsE8wFBeQRNLTva2Pb9PombyoQQsYhcWzEbsit -V74xmZZn9tsG+qltMvU0FFKDKl6i6I4xRmg48A0z4jP65GnSrpcP7Wmdo3VzQ1Rl4Elj3Ocq -EADniGFFbG2nJKtFK++m/oK2UPxJg5p1ks97vcoF+eQWBeqthZkt2Sc9nAqGUmCM7jiuUDPF -gaa2chj00oy+w3rhM8qxYKcvwg5ATTKRyz6b47kQxdXYU2akhlftaQgha2vv+5aYbWuVAtkj -3sDsBczYbUFqOUl3A+e2PQxTWnDbRUDXwSTw5s5QuwN/0h8z9ZYO/4y/wVdCS3ev1pJUuQIN -BFeEEGQBEACg3Og+IIMKAi1UUc4QxyDOluzb9yHO57dYivrD9e9ozTrA0vt/QD58wn+qLb3e -pd8hTkYZ8B4OJ6XFs9dMODpwBf7rnX4xwtzHnZu+bXnHEITljdwLabpVHrOW42/6Aek11oaK -zveRxjTC3848PH/OVbVOZNzrqNwijIuG5qJalEQUtdOodSBafQjOM5ftckDbqEXChiirqRNl -evb7r1uVQ/Jq7oLWPq0I/WkOSpIh93m6Cr8MwoI2W/zdhF8X42xLWwrZEEZsZjNC61x2rS8j -t5VZrl4QNMkLr4JqPQ0u7xZzDiiAj73hd6L0uyCw4SI+H5lhN4Ekm+1EJqXMM0kmG7QrPvzi -637WnporA/Ej7o400yguGKFynxG5gsVO6xMj6dcSMz2slFdFI/s8xoKhxddK69F/Ms/y1+yq -PYs0S2Rh9biOvfHxv0C0oggXT3M4u8kOn9M0nO5pqb/D9xKjVJUMIaW6RhB6QNF/982NXX1l -DqHr1hN60/hux1DzIt0U636oJn9w0GHMwEe5ls7W6aGACqqRUopDatcJaAZLp1kLzfw0yDLu -LPOYhPlcL726jYSBfWlxqOPvjgJy3s+C2oAryafkuRS0INTAVKfINjucFdFfH1g1MoHYlomw -bxHiA+iNeUJB7yvFF1bX+mh+csRyejzuk49EF7IyXgcopQARAQABiQREBBgBCAAPBQJXhBBk -AhsCBQkLRzUAAikJENOoDjA4LpTewV0gBBkBCAAGBQJXhBBkAAoJELyVRwG/9ikeROkP/R+/ -Yr3YVWUOh/feG7MQXjdEnnlH/03GdjUK7G9C15SFVz7JD+ku5DN28LE6x6PElWznMz3ym0VY -Ptlk8z6lBxS2G8DXdXKQO8ok7XmQJ2t7q/5jySwWgMhVWeUlLcr626In9H0SeXKM0y21EFx+ -nvNrskJl0/hXqli9XQxfSiMAi+3wgqeCuiaVhRp6DvEQZL9brNhMWgN0JDJNBNiw0FrLnBSF -fc3stfV1PTF1x4U3m7rOIBjiDX+E9baBSkTqDqtZf6Zontf8qnSP/n/4CtpE2do0cCBQbRAx -8BJdMnXfSs2uSeBRHG2VX4bn6y/nlug+WsRpbQ3AXU0GhPYuDL+ySVHU5kQPfmJ7R70+L+hQ -Jv2IRhHLDKQrUjBUmscJBUruii2tsBf5n8vn7WXa60L+WdIBIr5ZpFGRTQVMaZw6fcnVd7bq -+6ckZDJSsxiV0cOWGEv5GgPjRBVxNHA6yf+to8xfZSdufNrc+XVZ0y44YBANVhC0P/+Ci7X9 -TQD/UZiZAynoZwJvHbo1qKB7bsbqR1SJmyrcJ22yuJauPMilrwi+EexQRoLyt5KXMSekdzbm -4rn2tfMOSeDTeG/CjYbU+gFBDsD7QTHT7VXbtefeNxMOJmFo4Dv9SMYYGbAqZyxTeMV588rI -QB/tdihgzAdkQsw8lrvOTjdcXkb7kJLq7wkP/RFpwiY1YrhBYl4jDRI7fv032tAEiZOPb8Tb -a3ruZxdAkgYuk3HP7zDxSqVed3US+RFopK2TXuoRL+GWzftnjJDFfygtAq/8dtNCFO7yRvRU -LjSV6a/tYPW1+UnWvxg7O9q91OlZ3/OFWqC0hmFwyd4vYRTsN1HwAIG19+WKvlPUukINCE2p -a74ozwrFkmRbB3dieCfBUyTpL0uIzG0Azg2QKpKeFWl1KswU6BP3WppACcXEw4eXRpm6XO/g -dxyHT8U2AhchmezF3gCElRzP/i7By0qd0HDxo5pwZuWzuE6DVL42K8rQZfjbZYdSQeUuON5i -avPjqIUU4ghFpn7Tj/RoD9KBUSJnFSGTcNIPGRhxAibMM/GE7Hrf5HAxG3WRVYrdw08gy9LL -JGcPCSTECock80nlA+Npu232V2UYa9XAnjWrCincwCZkbT2OG7IYPtfH00DMffdTUkN23Nnd -RCqCssyf9hPSBV8Dm7PXdXd6lkJn9rEvr/b859Q6bd85H2K9pIgmKhkYD9ZkIAHDgTQ2J1T/ -OG3uVx6q+NX6VQWioDTAaBmXMvuSCZbcexr0i5SKUc+3mpRHhHF6voqDg/RE/FLsztR/6eWn -s9sfGiAt2Ye9vvIsSn703yfcYX4l02woMTZ4d43M7mD3NA8cj/sGfnQNlh9xpuI9M+8ZHCYJ -=VSVF ------END PGP PUBLIC KEY BLOCK----- diff --git a/etc/datadog.list b/etc/datadog.list index 03751bc..fab8e6f 100644 --- a/etc/datadog.list +++ b/etc/datadog.list @@ -1 +1 @@ -deb https://apt.datadoghq.com/ stable 6 +deb [signed-by=SIGNED_BY_PLACEHOLDER] https://apt.datadoghq.com/ stable 6 diff --git a/etc/datadog7.list b/etc/datadog7.list index e7c5946..ce8b852 100644 --- a/etc/datadog7.list +++ b/etc/datadog7.list @@ -1 +1 @@ -deb https://apt.datadoghq.com/ stable 7 +deb [signed-by=SIGNED_BY_PLACEHOLDER] https://apt.datadoghq.com/ stable 7 diff --git a/test/compile_and_run_test.sh b/test/compile_and_run_test.sh index d6067a5..4145e68 100644 --- a/test/compile_and_run_test.sh +++ b/test/compile_and_run_test.sh @@ -59,7 +59,9 @@ compileAndRunVersion() testReleased6Versions() { + echo "deb [trusted=yes] https://apt.datadoghq.com/ stable 6" > "${BUILDPACK_HOME}/etc/datadog.list" getAvailableVersions "datadog.list" + echo "deb [signed-by=SIGNED_BY_PLACEHOLDER] https://apt.datadoghq.com/ stable 6" > "${BUILDPACK_HOME}/etc/datadog.list" for VERSION in ${AGENT_VERSIONS}; do compileAndRunVersion $VERSION @@ -68,7 +70,9 @@ testReleased6Versions() testReleased7Versions() { + echo "deb [trusted=yes] https://apt.datadoghq.com/ stable 7" > "${BUILDPACK_HOME}/etc/datadog7.list" getAvailableVersions "datadog7.list" + echo "deb [signed-by=SIGNED_BY_PLACEHOLDER] https://apt.datadoghq.com/ stable 7" > "${BUILDPACK_HOME}/etc/datadog7.list" for VERSION in ${AGENT_VERSIONS}; do compileAndRunVersion $VERSION From 4e734c0ffc18cff6637e3962d9eeb911b638930d Mon Sep 17 00:00:00 2001 From: Jeremy Date: Thu, 20 May 2021 11:39:56 -0400 Subject: [PATCH 074/175] nits --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index b1cd39f..d415c77 100644 --- a/README.md +++ b/README.md @@ -296,9 +296,9 @@ Earlier versions of this project were forked from the [miketheman heroku-buildpa ## Troubleshooting -### Getting the agent status +### Getting the Agent status -If you have set up the buildpack and you are not getting some of the data you expected in Datadog, you can run the status command for the Datadog agent to help you finding the cause. +If you have set up the buildpack and you are not getting some of the data you expect in Datadog, you can run the status command for the Datadog Agent to help you find the cause. ```shell # Export the name of your Heroku application as an environment variable @@ -314,7 +314,7 @@ heroku ps:exec -a $APPNAME ~ $ ``` -You can ignore the warnings about the DD_API_KEY not being set. This is normal. The reason is that [Heroku doesn’t set configuration variables for the SSH session itself](https://devcenter.heroku.com/articles/exec#environment-variables), but the Datadog agent process was able to access those. +You can ignore the warnings about DD_API_KEY not being set. While [Heroku doesn't set configuration variables for the SSH session itself](https://devcenter.heroku.com/articles/exec#environment-variables), the Datadog Agent process is able to access them. Once inside the SSH session, execute the Datadog status command. @@ -331,7 +331,7 @@ Agent (v7.27.0) ``` -We will hightligth some of the sections you should focus on, depending on what you are trying to debug. +We will highligth some of the sections you should focus on, depending on what you are trying to debug. #### I am not getting any data in Datadog @@ -377,7 +377,7 @@ Collector #### Check APM agent -If you have instrumented your application for APM and not getting traces in Datadog, you can check that the APM agent is running correctly and collecting traces: +If you have instrumented your application for APM and are not getting traces in Datadog, you can check that the APM Agent is running correctly and collecting traces: ```` [...] @@ -406,7 +406,7 @@ APM Agent [...] ``` -### Datadog is reporting a higher number of agents than dynos +### Datadog is reporting a higher number of Agents than dynos Make sure you have `DD_DYNO_HOST` set to `true` and that `HEROKU_APP_NAME` has a value set for every Heroku application. See the [Hostname section](#hostname) for details. From b7af54a093c9459203d31c5bfd006a3d97d91fdc Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Tue, 25 May 2021 10:19:55 +0200 Subject: [PATCH 075/175] 7.28 and 6.28 as pinned versions --- bin/compile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/compile b/bin/compile index 79e1295..aeadc75 100644 --- a/bin/compile +++ b/bin/compile @@ -11,8 +11,8 @@ set -o pipefail # set -x # Set agent pinned version -DD_AGENT_PINNED_VERSION_6="6.27.0-1" -DD_AGENT_PINNED_VERSION_7="7.27.0-1" +DD_AGENT_PINNED_VERSION_6="6.28.0-1" +DD_AGENT_PINNED_VERSION_7="7.28.0-1" # Parse and derive params BUILD_DIR=$1 From 9c35d21dad8be81283fb875ce730dd7e7ace380a Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Fri, 28 May 2021 08:19:34 +0200 Subject: [PATCH 076/175] Fix README formatting --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b0b8e09..7b89f1e 100644 --- a/README.md +++ b/README.md @@ -387,7 +387,7 @@ Collector If you have instrumented your application for APM and are not getting traces in Datadog, you can check that the APM Agent is running correctly and collecting traces: -```` +``` [...] ========= APM Agent From 2f1221096c7eda929efe7e7754697dd5480f442e Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Tue, 25 May 2021 10:36:13 +0200 Subject: [PATCH 077/175] Prepare 1.26 release --- CHANGELOG.md | 8 ++++++++ extra/datadog.sh | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a64267..56985dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [1.26] - 2021-05-28 + +### Added +- Datadog agent pinned versions are now `6.28.0` and `7.28.0` + +### Changed +- Apt signing keys are now obtained from keys.datadoghq.com + ## [1.25] - 2021-04-19 ### Added diff --git a/extra/datadog.sh b/extra/datadog.sh index bc58fd2..834c75b 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -50,7 +50,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="dev" +BUILDPACKVERSION="1.26" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # We want always to have the Dyno ID as a host alias to improve correlation From ab550c0798b8f1647b2ba132571ea2ed68b49fed Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Fri, 28 May 2021 08:26:25 +0200 Subject: [PATCH 078/175] Revert buildpack version back to dev --- extra/datadog.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index 834c75b..bc58fd2 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -50,7 +50,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="1.26" +BUILDPACKVERSION="dev" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # We want always to have the Dyno ID as a host alias to improve correlation From 8c658341956edc02338c0af8fa358fddd048d1f6 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Mon, 28 Jun 2021 15:09:40 +0200 Subject: [PATCH 079/175] 7.29 and 6.29 as pinned versions --- bin/compile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/compile b/bin/compile index aeadc75..05da3b6 100644 --- a/bin/compile +++ b/bin/compile @@ -11,8 +11,8 @@ set -o pipefail # set -x # Set agent pinned version -DD_AGENT_PINNED_VERSION_6="6.28.0-1" -DD_AGENT_PINNED_VERSION_7="7.28.0-1" +DD_AGENT_PINNED_VERSION_6="6.29.0-1" +DD_AGENT_PINNED_VERSION_7="7.29.0-1" # Parse and derive params BUILD_DIR=$1 From a63f7b047f30712997f8f5ce424be7bc4fc9e9f5 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Mon, 28 Jun 2021 15:54:34 +0200 Subject: [PATCH 080/175] Buildpack 1.27 release --- CHANGELOG.md | 6 ++++++ extra/datadog.sh | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 56985dd..ecdf0ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [1.27] - 2021-06-28 + +### Added +- Datadog agent pinned versions are now `6.29.0` and `7.29.0` +- The Dyno ID is now sent as host alias (requires Agent 6.29.0/7.29.0 or newer) + ## [1.26] - 2021-05-28 ### Added diff --git a/extra/datadog.sh b/extra/datadog.sh index bc58fd2..183a72d 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -50,7 +50,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="dev" +BUILDPACKVERSION="1.27" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # We want always to have the Dyno ID as a host alias to improve correlation From 0ee6431625fe2a05d8ba44bb45cfd0e01be46cb9 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Mon, 28 Jun 2021 16:03:06 +0200 Subject: [PATCH 081/175] Revert buildpack version back to dev --- extra/datadog.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index 183a72d..bc58fd2 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -50,7 +50,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="1.27" +BUILDPACKVERSION="dev" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # We want always to have the Dyno ID as a host alias to improve correlation From eaccece2797ff116d90cc985c7cbe14b0ad56514 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Thu, 1 Jul 2021 09:44:09 +0200 Subject: [PATCH 082/175] Add pip and setuptools dependencies for python3 as well --- extra/datadog.sh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index bc58fd2..d0219f5 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -142,12 +142,10 @@ DD_PYTHONPATH="$DD_DIR/embedded/lib/$PYTHON_DIR/lib-tk:$DD_PYTHONPATH" DD_PYTHONPATH="$DD_DIR/embedded/lib/$PYTHON_DIR/lib-dynload:$DD_PYTHONPATH" DD_PYTHONPATH="$DD_DIR/bin/agent/dist:$DD_PYTHONPATH" -# For Python2 we need to add explicitely pip and setuptools dependencies -if [ "$DD_PYTHON_VERSION" = "2" ]; then - PIP_PATH=$(find "$DD_DIR/embedded/lib/$PYTHON_DIR/site-packages" -maxdepth 1 -name "pip*egg") - SETUPTOOLS_PATH=$(find "$DD_DIR/embedded/lib/$PYTHON_DIR/site-packages" -maxdepth 1 -name "setuptools*egg") - DD_PYTHONPATH="$DD_PYTHONPATH:$SETUPTOOLS_PATH:$PIP_PATH" -fi +# We need to add explicitely pip and setuptools dependencies +PIP_PATH=$(find "$DD_DIR/embedded/lib/$PYTHON_DIR/site-packages" -maxdepth 1 -name "pip*egg") +SETUPTOOLS_PATH=$(find "$DD_DIR/embedded/lib/$PYTHON_DIR/site-packages" -maxdepth 1 -name "setuptools*egg") +DD_PYTHONPATH="$DD_PYTHONPATH:$SETUPTOOLS_PATH:$PIP_PATH" # Export agent's PYTHONPATH be used by the agent-wrapper export DD_PYTHONPATH="$DD_DIR/embedded/lib:$DD_PYTHONPATH" From 39cd07a68be829ed45daeac82fbb51dc1e2bcd87 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Thu, 1 Jul 2021 09:59:01 +0200 Subject: [PATCH 083/175] Fix tests. Library folder has incresed a bit starting with 7.30 --- test/compile_and_run_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/compile_and_run_test.sh b/test/compile_and_run_test.sh index 4145e68..9f55fcd 100644 --- a/test/compile_and_run_test.sh +++ b/test/compile_and_run_test.sh @@ -51,7 +51,7 @@ compileAndRunVersion() if test "$BIGSIZE_VERSION" = "$(echo "$BIGSIZE_VERSION\n$1" | sort -V | head -n1)" ; then assertTrue "Binary folder is too big: ${SIZE_BIN}" "[ $SIZE_BIN -lt 105000 ]" fi - assertTrue "Embedded library folder is too big: ${SIZE_LIB}" "[ $SIZE_LIB -lt 230000 ]" + assertTrue "Embedded library folder is too big: ${SIZE_LIB}" "[ $SIZE_LIB -lt 255000 ]" assertTrue "Embedded binary folder is too big: ${SIZE_EMB_BIN}" "[ $SIZE_EMB_BIN -lt 115000 ]" rm -rf $HOME/.apt/ From bc77db9b1c3fe50fa9b7fc822472ebc0374b454c Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Thu, 1 Jul 2021 16:37:43 +0200 Subject: [PATCH 084/175] Preparation for release 1.28 --- CHANGELOG.md | 5 +++++ extra/datadog.sh | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ecdf0ca..f4b7627 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [1.28] - 2021-07-01 + +### Fixed +- Fixed memcached integration for Python 3 + ## [1.27] - 2021-06-28 ### Added diff --git a/extra/datadog.sh b/extra/datadog.sh index d0219f5..0a68893 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -50,7 +50,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="dev" +BUILDPACKVERSION="1.28" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # We want always to have the Dyno ID as a host alias to improve correlation From 826882af7a231d431da72ca0720b459ccf88720d Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Thu, 1 Jul 2021 16:58:35 +0200 Subject: [PATCH 085/175] Revert buildpack version back to dev --- extra/datadog.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index 0a68893..d0219f5 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -50,7 +50,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="1.28" +BUILDPACKVERSION="dev" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # We want always to have the Dyno ID as a host alias to improve correlation From 4782ba303d0881df7609bdaad59198f5650e2e44 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 28 Jul 2021 11:37:17 +0200 Subject: [PATCH 086/175] Improve documentation related to disabling the agent for certain type of dynos --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index 7b89f1e..74ee227 100644 --- a/README.md +++ b/README.md @@ -120,6 +120,19 @@ Depending on your use case, you may want to set your hostname so that hosts are For this to work correctly, `HEROKU_APP_NAME` needs to be set. The easiest way to do this is by [enabling dyno metadata][13]. **Note**: Dyno metadata is not yet available in Private Spaces, in which case you need to set `HEROKU_APP_NAME` manually. +## Disabling the Datadog Agent for short-lived dynos + +By default, the Datadog Agent will run on each of the dynos that are part of the application. This includes `scheduler`, `release` or `run` dynos. In many cases the metrics from these dynos are not needed and it makes sense to disable the Datadog Agent for those. + +To disable the Datadog Agent based on dyno type, add the following snippet to your [prerun.sh script](#prerun-script) (adapting it to the type of dynos you don't want to monitor): + +```shell +# Disable the Datadog Agent based on dyno type +if [ "$DYNOTYPE" == "run" ] || [ "$DYNOTYPE" == "scheduler" ] || [ "$DYNOTYPE" == "release" ]; then + DISABLE_DATADOG_AGENT="true" +fi +``` + ## System metrics By default, the buildpack collects system metrics for the host machine running your dyno. System metrics are not available for individual dynos using this buildpack. To disable host system metrics collection, set the `DD_DISABLE_HOST_METRICS` environment variable to `true`. From 158acea1cc7023beae4f9e5378f247b4ddebbd20 Mon Sep 17 00:00:00 2001 From: Ruth Naebeck Date: Fri, 30 Jul 2021 16:33:14 -0400 Subject: [PATCH 087/175] DOCS-2275 Lint README --- README.md | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 74ee227..bb8701f 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ This [Heroku buildpack][1] installs the Datadog Agent in your Heroku dyno to col ## Installation -This guide assumes that you already have your application running on Heroku. Please, refer to Heroku documentation to learn how to deploy your application to Heroku. +This guide assumes that you already have your application running on Heroku. See the Heroku documentation to learn how to deploy your application to Heroku. 1. Go to [Datadog API settings][3] and copy your Datadog API key. Export it to an environment variable: @@ -43,9 +43,9 @@ Once complete, the Datadog Agent is started automatically when each dyno starts. The Datadog Agent provides a listening port on `8125` for statsd/dogstatsd metrics and events. Traces are collected on port `8126`. ### Order of buildpacks -As explained in [Heroku's documentation][4], the last buildpack in the list is used to determine the process type for the application. +As explained in the [Heroku][4] documentation, the last buildpack in the list is used to determine the process type for the application. -Buildpacks that install apt packages (e.g. [heroku-buildpack-apt][5], [puppeteer-heroku-buildpack][6]) or buildpacks that modify the `/app` folder (e.g. [heroku-buildpack-monorepo][7]) need to be added **before** the Datadog buildpack. For example, if your application uses the `ruby`, `datadog` and `apt` buildpacks, this would be a correct `heroku buildpacks` output: +Buildpacks that install apt packages, such as [heroku-buildpack-apt][5], [puppeteer-heroku-buildpack][6], or buildpacks that modify the `/app` folder, such as [heroku-buildpack-monorepo][7], need to be added **before** the Datadog buildpack. For example, if your application uses the `ruby`, `datadog` and `apt` buildpacks, this would be a correct `heroku buildpacks` output: ```text 1. https://github.com/heroku/heroku-buildpack-apt.git @@ -90,39 +90,39 @@ git push heroku master ## Configuration -In addition to the environment variables shown above, there are a number of others you can set: +In addition to the environment variables shown above, there are several others you can set: | Setting | Description | |----------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `DD_API_KEY` | *Required.* Your API key is available from the [Datadog API Integrations][9] page. Note that this is the *API* key, not the application key. | +| `DD_API_KEY` | *Required.* Your API key is available from the [Datadog API Integrations][9] page. **Note**: This is the *API* key, not the application key. | | `DD_HOSTNAME` | *Optional.* **WARNING**: Setting the hostname manually may result in metrics continuity errors. It is recommended that you do *not* set this variable. Because dyno hosts are ephemeral it is recommended that you monitor based on the tags `dynoname` or `appname`. | -| `DD_DYNO_HOST` | *Optional.* Set to `true` to use the dyno name (e.g. `web.1` or `run.1234`) as the hostname. See the [hostname section](#hostname) below for more information. Defaults to `false` | -| `DD_TAGS` | *Optional.* Sets additional tags provided as a space-separated string (**Note**: comma-separated string in buildpack versions `1.16` and earlier; this is still supported to maintain backward compatibility). For example, `heroku config:set DD_TAGS="simple-tag-0 tag-key-1:tag-value-1"`. The buildpack automatically adds the tags `dyno` which represent the dyno name (e.g. web.1) and `dynotype` (the type of dyno, e.g `run` or `web`). See the ["Guide to tagging"][10] for more information. | +| `DD_DYNO_HOST` | *Optional.* Set to `true` to use the dyno name, such as `web.1` or `run.1234`, as the hostname. See the [hostname section](#hostname) below for more information. Defaults to `false` | +| `DD_TAGS` | *Optional.* Sets additional tags provided as a space-separated string (**Note**: comma-separated string in buildpack versions `1.16` and earlier; this is still supported to maintain backward compatibility). For example, `heroku config:set DD_TAGS="simple-tag-0 tag-key-1:tag-value-1"`. The buildpack automatically adds the tags `dyno` which represent the dyno name, such as web.1, and `dynotype` (the type of dyno, e.g `run` or `web`). See the ["Guide to tagging"][10] for more information. | | `DD_VERSION` | *Optional.* Sets the version of your application, used to organize traces by version. | | `DD_HISTOGRAM_PERCENTILES` | *Optional.* Optionally set additional percentiles for your histogram metrics. See [How to graph percentiles][11]. | | `DISABLE_DATADOG_AGENT` | *Optional.* When set, the Datadog Agent does not run. | | `DD_APM_ENABLED` | *Optional.* Trace collection is enabled by default. Set this to `false` to disable trace collection. Changing this option requires recompilation of the slug. Check [the upgrading and slug recompilation section](#upgrading-and-slug-recompilation) for details. | | `DD_PROCESS_AGENT` | *Optional.* The Datadog Process Agent is disabled by default. Set this to `true` to enable the Process Agent. Changing this option requires recompilation of the slug. Check [the upgrading and slug recompilation section](#upgrading-and-slug-recompilation) for details. | | `DD_SITE` | *Optional.* If you use the app.datadoghq.eu service, set this to `datadoghq.eu`. Defaults to `datadoghq.com`. | -| `DD_AGENT_VERSION` | *Optional.* By default, the buildpack installs the latest 6.x version of the Datadog Agent available in the package repository. Use this variable to install older versions of the Datadog Agent (note that not all versions of the Agent may be available). This option takes precedence over `DD_AGENT_MAJOR_VERSION`. Changing this option requires recompiling the slug. Check [the upgrading and slug recompilation section](#upgrading-and-slug-recompilation) for details. | +| `DD_AGENT_VERSION` | *Optional.* By default, the buildpack installs the latest 6.x version of the Datadog Agent available in the package repository. Use this variable to install older versions of the Datadog Agent. **Note**: Not all versions of the Agent may be available. This option takes precedence over `DD_AGENT_MAJOR_VERSION`. Changing this option requires recompiling the slug. Check [the upgrading and slug recompilation section](#upgrading-and-slug-recompilation) for details. | | `DD_AGENT_MAJOR_VERSION` | *Optional.* By default, the buildpack installs the latest 6.x version of the Datadog Agent available in the package repository. Set this variable to `7` to install the latest 7.x version of the Datadog Agent. Check the [Python versions section](#python-and-agent-versions) for more information on the relation of the Agent version and Python version. Changing this option requires recompiling the slug. Check [the upgrading and slug recompilation section](#upgrading-and-slug-recompilation) for details. | | `DD_DISABLE_HOST_METRICS` | *Optional.* By default, the buildpack reports system metrics for the host machine running the dyno. Set this to `true` to disable system metrics collection. See the [system metrics section](#system-metrics) below for more information. | | `DD_PYTHON_VERSION` | *Optional.* Starting with version `6.14.0`, Datadog Agent ships with Python versions `2` and `3`. The buildpack only keeps one of the versions. Set this to `2` or `3` to select the Python version you want the Agent to keep. If not set, the buildpack keeps `2`. Check the [Python versions section](#python-and-agent-versions) for more information. Changing this option requires recompiling the slug. Check [the upgrading and slug recompilation section](#upgrading-and-slug-recompilation) for details. | -| `DD_HEROKU_CONF_FOLDER` | *Optional.* By default, the buildpack looks in the root of your application for a folder `/datadog` for any configuration files you wish to include, eg see [prerun.sh script](#prerun-script). This location can be overridden by setting this to your desired path. | +| `DD_HEROKU_CONF_FOLDER` | *Optional.* By default, the buildpack looks in the root of your application for a folder `/datadog` for any configuration files you wish to include, see [prerun.sh script](#prerun-script). This location can be overridden by setting this to your desired path. | -For additional documentation, refer to the [Datadog Agent documentation][12]. +For additional documentation, see the [Datadog Agent][12] documentation. ## Hostname Heroku dynos are ephemeral—they can move to different host machines whenever new code is deployed, configuration changes are made, or resouce needs/availability changes. This makes Heroku flexible and responsive, but can potentially lead to a high number of reported hosts in Datadog. Datadog bills on a per-host basis, and the buildpack default is to report actual hosts, which can lead to higher than expected costs. -Depending on your use case, you may want to set your hostname so that hosts are aggregated and report a lower number. To do this, Set `DD_DYNO_HOST` to `true`. This causes the Agent to report the hostname as the app and dyno name (for example `appname.web.1` or `appname.run.1234`) and your host count will closely match your dyno usage. One drawback is that you may see some metrics continuity errors whenever a dyno is cycled. +Depending on your use case, you may want to set your hostname so that hosts are aggregated and report a lower number. To do this, Set `DD_DYNO_HOST` to `true`. This causes the Agent to report the hostname as the app and dyno name (for example `appname.web.1` or `appname.run.1234`) and your host count closely matches your dyno usage. One drawback is that you may see some metrics continuity errors whenever a dyno is cycled. For this to work correctly, `HEROKU_APP_NAME` needs to be set. The easiest way to do this is by [enabling dyno metadata][13]. **Note**: Dyno metadata is not yet available in Private Spaces, in which case you need to set `HEROKU_APP_NAME` manually. ## Disabling the Datadog Agent for short-lived dynos -By default, the Datadog Agent will run on each of the dynos that are part of the application. This includes `scheduler`, `release` or `run` dynos. In many cases the metrics from these dynos are not needed and it makes sense to disable the Datadog Agent for those. +By default, the Datadog Agent runs on each of the dynos that are part of the application. This includes `scheduler`, `release` or `run` dynos. In many cases the metrics from these dynos are not needed and it makes sense to disable the Datadog Agent for those. To disable the Datadog Agent based on dyno type, add the following snippet to your [prerun.sh script](#prerun-script) (adapting it to the type of dynos you don't want to monitor): @@ -187,7 +187,7 @@ agent-wrapper integration install -t datadog-ping==1.0.0 ## Prerun script -In addition to all of the configurations above, you can include a prerun script, `/datadog/prerun.sh`, in your application. The prerun script runs after all of the standard configuration actions and immediately before starting the Datadog Agent. This allows you to modify the environment variables (for example, DD_TAGS or DD_VERSION), perform additional configurations, install community integrations, or even disable the Datadog Agent programmatically. +In addition to all of the configurations above, you can include a prerun script, `/datadog/prerun.sh`, in your application. The prerun script runs after all of the standard configuration actions and immediately before starting the Datadog Agent. This allows you to modify the environment variables (for example, DD_TAGS, or DD_VERSION), perform additional configurations, install community integrations, or even disable the Datadog Agent programmatically. The example below demonstrates a few of the things you can do in the `prerun.sh` script: @@ -238,7 +238,7 @@ To reduce your slug size, make sure that `DD_APM_ENABLED` is set to `false`, if ## Debugging -To run any of the information/debugging commands listed in the [Agent's documentation][22] use the `agent-wrapper` command. +To run any of the information/debugging commands listed in the [Agent][22] documentation use the `agent-wrapper` command. For example, to display the status of your Datadog Agent and enabled integrations, run: @@ -248,9 +248,9 @@ agent-wrapper status ## Python and Agent versions -Prior to version `6.14` the Datadog v6 Agent shipped with Python version `2` embedded. Starting with `6.14`, and in preparation for Python version `2` End Of Life, announced for January 2020, the Datadog v6 Agent ships with both Python versions `2` and `3`, to give customers enough time to migrate their custom checks to Python version `3`. The Heroku buildpack only keeps one of the versions. Set `DD_PYTHON_VERSION` to `2` or `3` to select the Python version you want the Agent to keep. If not set, the buildpack keeps Python version `2`. If you are using custom checks that only work with Python version `2`, we recommend to migrate them to version `3` before its EOL. +Prior to version `6.14` the Datadog v6 Agent shipped with Python version `2` embedded. Starting with `6.14`, and in preparation for Python version `2` End Of Life, announced for January 2020, the Datadog v6 Agent ships with both Python versions `2` and `3`, to give customers enough time to migrate their custom checks to Python version `3`. The Heroku buildpack only keeps one of the versions. Set `DD_PYTHON_VERSION` to `2` or `3` to select the Python version you want the Agent to keep. If not set, the buildpack keeps Python version `2`. If you are using custom checks that only work with Python version `2`, Datadog recommends migrating them to version `3` before its EOL. -Agent v7 only ships with Python version `3`. If you are not using custom checks or your custom checks are already migrated to version `3`, we recommend moving to Agent 7 as soon as possible. Starting with `6.15`, v7 releases with the same minor version share the same feature set, making it safe to move between those two. For example, if you are running `6.16` and you don't need Python version `2`, it is safe to jump to `7.16`. +Agent v7 only ships with Python version `3`. If you are not using custom checks or your custom checks are already migrated to version `3`, Datadog recommends moving to Agent v7 as soon as possible. Starting with `6.15`, v7 releases with the same minor version share the same feature set, making it safe to move between those two. For example, if you are running `6.16` and you don't need Python version `2`, it is safe to jump to `7.16`. ## Heroku log collection @@ -309,7 +309,7 @@ For more advanced options in the Docker image, reference the [Datadog Agent Dock ## Contributing -See the [contributing documentation][25] to learn how to open an issue or PR to the [Heroku-buildpack-datadog repository][26]. +See the [contributing][25] documentation to learn how to open an issue or PR to the [Heroku-buildpack-datadog repository][26]. ## History @@ -352,9 +352,9 @@ Agent (v7.27.0) ``` -We will highligth some of the sections you should focus on, depending on what you are trying to debug. +### Debugging -#### I am not getting any data in Datadog +#### No data in Datadog Make sure that the `status` command runs correctly and that this section of the output tells you that your API key is valid: From 4f86e7abbbb2860924eaf51416ac5ef9c91da5fa Mon Sep 17 00:00:00 2001 From: ruthnaebeck <19349244+ruthnaebeck@users.noreply.github.com> Date: Tue, 3 Aug 2021 11:47:12 -0400 Subject: [PATCH 088/175] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bb8701f..5261e69 100644 --- a/README.md +++ b/README.md @@ -187,7 +187,7 @@ agent-wrapper integration install -t datadog-ping==1.0.0 ## Prerun script -In addition to all of the configurations above, you can include a prerun script, `/datadog/prerun.sh`, in your application. The prerun script runs after all of the standard configuration actions and immediately before starting the Datadog Agent. This allows you to modify the environment variables (for example, DD_TAGS, or DD_VERSION), perform additional configurations, install community integrations, or even disable the Datadog Agent programmatically. +In addition to all of the configurations above, you can include a prerun script, `/datadog/prerun.sh`, in your application. The prerun script runs after all of the standard configuration actions and immediately before starting the Datadog Agent. This allows you to modify the environment variables (for example: DD_TAGS or DD_VERSION), perform additional configurations, install community integrations, or even disable the Datadog Agent programmatically. The example below demonstrates a few of the things you can do in the `prerun.sh` script: From 6a0fa9d9ddeccf531935fcbc646ae324defc2f10 Mon Sep 17 00:00:00 2001 From: Ruth Naebeck Date: Wed, 4 Aug 2021 12:03:59 -0400 Subject: [PATCH 089/175] updates for comments --- README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 5261e69..20e286b 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ Once complete, the Datadog Agent is started automatically when each dyno starts. The Datadog Agent provides a listening port on `8125` for statsd/dogstatsd metrics and events. Traces are collected on port `8126`. ### Order of buildpacks -As explained in the [Heroku][4] documentation, the last buildpack in the list is used to determine the process type for the application. +As explained in the Heroku documentation under [Viewing buildpacks][4], the last buildpack in the list is used to determine the process type for the application. Buildpacks that install apt packages, such as [heroku-buildpack-apt][5], [puppeteer-heroku-buildpack][6], or buildpacks that modify the `/app` folder, such as [heroku-buildpack-monorepo][7], need to be added **before** the Datadog buildpack. For example, if your application uses the `ruby`, `datadog` and `apt` buildpacks, this would be a correct `heroku buildpacks` output: @@ -97,20 +97,20 @@ In addition to the environment variables shown above, there are several others y | `DD_API_KEY` | *Required.* Your API key is available from the [Datadog API Integrations][9] page. **Note**: This is the *API* key, not the application key. | | `DD_HOSTNAME` | *Optional.* **WARNING**: Setting the hostname manually may result in metrics continuity errors. It is recommended that you do *not* set this variable. Because dyno hosts are ephemeral it is recommended that you monitor based on the tags `dynoname` or `appname`. | | `DD_DYNO_HOST` | *Optional.* Set to `true` to use the dyno name, such as `web.1` or `run.1234`, as the hostname. See the [hostname section](#hostname) below for more information. Defaults to `false` | -| `DD_TAGS` | *Optional.* Sets additional tags provided as a space-separated string (**Note**: comma-separated string in buildpack versions `1.16` and earlier; this is still supported to maintain backward compatibility). For example, `heroku config:set DD_TAGS="simple-tag-0 tag-key-1:tag-value-1"`. The buildpack automatically adds the tags `dyno` which represent the dyno name, such as web.1, and `dynotype` (the type of dyno, e.g `run` or `web`). See the ["Guide to tagging"][10] for more information. | +| `DD_TAGS` | *Optional.* Sets additional tags provided as a space-separated string (**Note**: comma-separated string in buildpack versions `1.16` and earlier; this is still supported to maintain backward compatibility). For example, `heroku config:set DD_TAGS="simple-tag-0 tag-key-1:tag-value-1"`. The buildpack automatically adds the tags `dyno` which represent the dyno name, such as `web.1`, and `dynotype` (the type of dyno, e.g `run` or `web`). See the [Guide to tagging][10] for more information. | | `DD_VERSION` | *Optional.* Sets the version of your application, used to organize traces by version. | | `DD_HISTOGRAM_PERCENTILES` | *Optional.* Optionally set additional percentiles for your histogram metrics. See [How to graph percentiles][11]. | | `DISABLE_DATADOG_AGENT` | *Optional.* When set, the Datadog Agent does not run. | | `DD_APM_ENABLED` | *Optional.* Trace collection is enabled by default. Set this to `false` to disable trace collection. Changing this option requires recompilation of the slug. Check [the upgrading and slug recompilation section](#upgrading-and-slug-recompilation) for details. | | `DD_PROCESS_AGENT` | *Optional.* The Datadog Process Agent is disabled by default. Set this to `true` to enable the Process Agent. Changing this option requires recompilation of the slug. Check [the upgrading and slug recompilation section](#upgrading-and-slug-recompilation) for details. | | `DD_SITE` | *Optional.* If you use the app.datadoghq.eu service, set this to `datadoghq.eu`. Defaults to `datadoghq.com`. | -| `DD_AGENT_VERSION` | *Optional.* By default, the buildpack installs the latest 6.x version of the Datadog Agent available in the package repository. Use this variable to install older versions of the Datadog Agent. **Note**: Not all versions of the Agent may be available. This option takes precedence over `DD_AGENT_MAJOR_VERSION`. Changing this option requires recompiling the slug. Check [the upgrading and slug recompilation section](#upgrading-and-slug-recompilation) for details. | -| `DD_AGENT_MAJOR_VERSION` | *Optional.* By default, the buildpack installs the latest 6.x version of the Datadog Agent available in the package repository. Set this variable to `7` to install the latest 7.x version of the Datadog Agent. Check the [Python versions section](#python-and-agent-versions) for more information on the relation of the Agent version and Python version. Changing this option requires recompiling the slug. Check [the upgrading and slug recompilation section](#upgrading-and-slug-recompilation) for details. | +| `DD_AGENT_VERSION` | *Optional.* By default, the buildpack installs the latest 6.x version of the Datadog Agent available in the package repository. Use this variable to install older versions of the Datadog Agent. **Note**: Not all versions of the Agent may be available. This option takes precedence over `DD_AGENT_MAJOR_VERSION`. Changing this option requires recompiling the slug. See [upgrading and slug recompilation](#upgrading-and-slug-recompilation) for details. | +| `DD_AGENT_MAJOR_VERSION` | *Optional.* By default, the buildpack installs the latest 6.x version of the Datadog Agent available in the package repository. Set this variable to `7` to install the latest 7.x version of the Datadog Agent. Check the [Python versions section](#python-and-agent-versions) for more information on the relation of the Agent version and Python version. Changing this option requires recompiling the slug. See [upgrading and slug recompilation](#upgrading-and-slug-recompilation) for details. | | `DD_DISABLE_HOST_METRICS` | *Optional.* By default, the buildpack reports system metrics for the host machine running the dyno. Set this to `true` to disable system metrics collection. See the [system metrics section](#system-metrics) below for more information. | | `DD_PYTHON_VERSION` | *Optional.* Starting with version `6.14.0`, Datadog Agent ships with Python versions `2` and `3`. The buildpack only keeps one of the versions. Set this to `2` or `3` to select the Python version you want the Agent to keep. If not set, the buildpack keeps `2`. Check the [Python versions section](#python-and-agent-versions) for more information. Changing this option requires recompiling the slug. Check [the upgrading and slug recompilation section](#upgrading-and-slug-recompilation) for details. | | `DD_HEROKU_CONF_FOLDER` | *Optional.* By default, the buildpack looks in the root of your application for a folder `/datadog` for any configuration files you wish to include, see [prerun.sh script](#prerun-script). This location can be overridden by setting this to your desired path. | -For additional documentation, see the [Datadog Agent][12] documentation. +For additional documentation, see the [Datadog Agent documentation][12]. ## Hostname @@ -238,7 +238,7 @@ To reduce your slug size, make sure that `DD_APM_ENABLED` is set to `false`, if ## Debugging -To run any of the information/debugging commands listed in the [Agent][22] documentation use the `agent-wrapper` command. +To run any of the [information or debugging commands][22], use the `agent-wrapper` command. For example, to display the status of your Datadog Agent and enabled integrations, run: @@ -248,9 +248,9 @@ agent-wrapper status ## Python and Agent versions -Prior to version `6.14` the Datadog v6 Agent shipped with Python version `2` embedded. Starting with `6.14`, and in preparation for Python version `2` End Of Life, announced for January 2020, the Datadog v6 Agent ships with both Python versions `2` and `3`, to give customers enough time to migrate their custom checks to Python version `3`. The Heroku buildpack only keeps one of the versions. Set `DD_PYTHON_VERSION` to `2` or `3` to select the Python version you want the Agent to keep. If not set, the buildpack keeps Python version `2`. If you are using custom checks that only work with Python version `2`, Datadog recommends migrating them to version `3` before its EOL. +Prior to version `6.14` the Datadog v6 Agent shipped with Python version `2` embedded. Starting with `6.14`, and in preparation for Python version `2` End Of Life, announced for January 2020, the Datadog v6 Agent ships with both Python versions `2` and `3`, to give customers enough time to migrate their custom checks to Python version `3`. The Heroku buildpack only keeps one of the versions. Set `DD_PYTHON_VERSION` to `2` or `3` to select the Python version you want the Agent to keep. If not set, the buildpack keeps Python version `2`. If you are using custom checks that only work with Python version `2`, migrate them to version `3` before its EOL. -Agent v7 only ships with Python version `3`. If you are not using custom checks or your custom checks are already migrated to version `3`, Datadog recommends moving to Agent v7 as soon as possible. Starting with `6.15`, v7 releases with the same minor version share the same feature set, making it safe to move between those two. For example, if you are running `6.16` and you don't need Python version `2`, it is safe to jump to `7.16`. +Agent v7 only ships with Python version `3`. If you are not using custom checks or your custom checks are already migrated to version `3`, move to Agent v7 as soon as possible. Starting with `6.15`, v7 releases with the same minor version share the same feature set, making it safe to move between those two. For example, if you are running `6.16` and you don't need Python version `2`, it is safe to jump to `7.16`. ## Heroku log collection @@ -309,7 +309,7 @@ For more advanced options in the Docker image, reference the [Datadog Agent Dock ## Contributing -See the [contributing][25] documentation to learn how to open an issue or PR to the [Heroku-buildpack-datadog repository][26]. +See the [contributing guidelines][25] to learn how to open an issue or PR to the [Heroku-buildpack-datadog repository][26]. ## History From d6ee50fa1a891f2eb626fc65b0cafcc24f3d3c43 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Thu, 23 Sep 2021 14:58:19 +0100 Subject: [PATCH 090/175] Pin agent 7.31 --- bin/compile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/compile b/bin/compile index 05da3b6..d9c7951 100644 --- a/bin/compile +++ b/bin/compile @@ -11,8 +11,8 @@ set -o pipefail # set -x # Set agent pinned version -DD_AGENT_PINNED_VERSION_6="6.29.0-1" -DD_AGENT_PINNED_VERSION_7="7.29.0-1" +DD_AGENT_PINNED_VERSION_6="6.31.0-1" +DD_AGENT_PINNED_VERSION_7="7.31.0-1" # Parse and derive params BUILD_DIR=$1 From 90ec35d97ea5132147174e5d2154bf8c53f54c73 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Thu, 23 Sep 2021 15:45:44 +0100 Subject: [PATCH 091/175] Remove unneeded static libraries --- bin/compile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bin/compile b/bin/compile index d9c7951..866b360 100644 --- a/bin/compile +++ b/bin/compile @@ -186,6 +186,9 @@ done rm -rf "$APT_DIR/opt/datadog-agent/embedded/bin/system-probe" rm -rf "$APT_DIR"/opt/datadog-agent/embedded/lib/libbcc* +# Remove static libraries +rm -rf "$APT_DIR"/opt/datadog-agent/embedded/lib/*.a + # Remove the Security agent rm -rf "$APT_DIR/opt/datadog-agent/embedded/bin/security-agent" From 71699d5ca469a67aaf48ef4e3970a5fe4d485137 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Thu, 23 Sep 2021 17:12:50 +0100 Subject: [PATCH 092/175] Prepare release 1.29 --- CHANGELOG.md | 5 +++++ extra/datadog.sh | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f4b7627..d3020d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [1.29] - 2021-09-23 + +### Added +- Datadog agent pinned versions are now `6.31.0` and `7.31.0` + ## [1.28] - 2021-07-01 ### Fixed diff --git a/extra/datadog.sh b/extra/datadog.sh index d0219f5..1ff4066 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -50,7 +50,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="dev" +BUILDPACKVERSION="1.29" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # We want always to have the Dyno ID as a host alias to improve correlation From 4967ef86fff7da7fc4d6e1a5ea8835c272ee8775 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Thu, 23 Sep 2021 17:17:52 +0100 Subject: [PATCH 093/175] Revert buildpack version back to dev --- extra/datadog.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index 1ff4066..d0219f5 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -50,7 +50,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="1.29" +BUILDPACKVERSION="dev" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # We want always to have the Dyno ID as a host alias to improve correlation From 49415c22abd27a5700037135376244a0fd02560a Mon Sep 17 00:00:00 2001 From: Ruth Naebeck Date: Thu, 21 Oct 2021 19:36:18 -0400 Subject: [PATCH 094/175] DOCS-2614 Update Datadog API page link --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 20e286b..96e0c49 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ In addition to the environment variables shown above, there are several others y | Setting | Description | |----------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `DD_API_KEY` | *Required.* Your API key is available from the [Datadog API Integrations][9] page. **Note**: This is the *API* key, not the application key. | +| `DD_API_KEY` | *Required.* Your API key is available from the [Datadog API Integrations][3] page. **Note**: This is the *API* key, not the application key. | | `DD_HOSTNAME` | *Optional.* **WARNING**: Setting the hostname manually may result in metrics continuity errors. It is recommended that you do *not* set this variable. Because dyno hosts are ephemeral it is recommended that you monitor based on the tags `dynoname` or `appname`. | | `DD_DYNO_HOST` | *Optional.* Set to `true` to use the dyno name, such as `web.1` or `run.1234`, as the hostname. See the [hostname section](#hostname) below for more information. Defaults to `false` | | `DD_TAGS` | *Optional.* Sets additional tags provided as a space-separated string (**Note**: comma-separated string in buildpack versions `1.16` and earlier; this is still supported to maintain backward compatibility). For example, `heroku config:set DD_TAGS="simple-tag-0 tag-key-1:tag-value-1"`. The buildpack automatically adds the tags `dyno` which represent the dyno name, such as `web.1`, and `dynotype` (the type of dyno, e.g `run` or `web`). See the [Guide to tagging][10] for more information. | @@ -437,13 +437,12 @@ After an upgrade of the buildpack or Agent, you must recompile your application' [1]: https://devcenter.heroku.com/articles/buildpacks [2]: https://docs.datadoghq.com/libraries -[3]: https://app.datadoghq.com/account/settings#api +[3]: https://app.datadoghq.com/organization-settings/api-keys [4]: https://devcenter.heroku.com/articles/using-multiple-buildpacks-for-an-app#viewing-buildpacks [5]: https://github.com/heroku/heroku-buildpack-apt [6]: https://github.com/jontewks/puppeteer-heroku-buildpack [7]: https://github.com/lstoll/heroku-buildpack-monorepo [8]: https://github.com/DataDog/heroku-buildpack-datadog/releases -[9]: https://app.datadoghq.com/account/settings#api [10]: https://docs.datadoghq.com/tagging/ [11]: https://docs.datadoghq.com/dashboards/guide/how-to-graph-percentiles-in-datadog/ [12]: https://docs.datadoghq.com/agent From 8789f8f0b8b396a77a6fd31de8ff000e311885d3 Mon Sep 17 00:00:00 2001 From: ruthnaebeck <19349244+ruthnaebeck@users.noreply.github.com> Date: Fri, 22 Oct 2021 10:16:06 -0400 Subject: [PATCH 095/175] Update README.md Co-authored-by: Ara Pulido --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 96e0c49..f826728 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ In addition to the environment variables shown above, there are several others y | Setting | Description | |----------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `DD_API_KEY` | *Required.* Your API key is available from the [Datadog API Integrations][3] page. **Note**: This is the *API* key, not the application key. | +| `DD_API_KEY` | *Required.* Your API key is available from the [Organization Settings -> API Keys][3] page. **Note**: This is the *API* key, not the application key. | | `DD_HOSTNAME` | *Optional.* **WARNING**: Setting the hostname manually may result in metrics continuity errors. It is recommended that you do *not* set this variable. Because dyno hosts are ephemeral it is recommended that you monitor based on the tags `dynoname` or `appname`. | | `DD_DYNO_HOST` | *Optional.* Set to `true` to use the dyno name, such as `web.1` or `run.1234`, as the hostname. See the [hostname section](#hostname) below for more information. Defaults to `false` | | `DD_TAGS` | *Optional.* Sets additional tags provided as a space-separated string (**Note**: comma-separated string in buildpack versions `1.16` and earlier; this is still supported to maintain backward compatibility). For example, `heroku config:set DD_TAGS="simple-tag-0 tag-key-1:tag-value-1"`. The buildpack automatically adds the tags `dyno` which represent the dyno name, such as `web.1`, and `dynotype` (the type of dyno, e.g `run` or `web`). See the [Guide to tagging][10] for more information. | From d605a1e44a5028cae381eb0c3c387b354e97312a Mon Sep 17 00:00:00 2001 From: Adam Ouellette Date: Mon, 29 Nov 2021 13:49:38 -0600 Subject: [PATCH 096/175] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f826728..304bdfc 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ For additional documentation, see the [Datadog Agent documentation][12]. ## Hostname -Heroku dynos are ephemeral—they can move to different host machines whenever new code is deployed, configuration changes are made, or resouce needs/availability changes. This makes Heroku flexible and responsive, but can potentially lead to a high number of reported hosts in Datadog. Datadog bills on a per-host basis, and the buildpack default is to report actual hosts, which can lead to higher than expected costs. +Heroku dynos are ephemeral—they can move to different host machines whenever new code is deployed, configuration changes are made, or resource needs/availability changes. This makes Heroku flexible and responsive, but can potentially lead to a high number of reported hosts in Datadog. Datadog bills on a per-host basis, and the buildpack default is to report actual hosts, which can lead to higher than expected costs. Depending on your use case, you may want to set your hostname so that hosts are aggregated and report a lower number. To do this, Set `DD_DYNO_HOST` to `true`. This causes the Agent to report the hostname as the app and dyno name (for example `appname.web.1` or `appname.run.1234`) and your host count closely matches your dyno usage. One drawback is that you may see some metrics continuity errors whenever a dyno is cycled. From 960bc46a59ab2be11a7b6edc77f08d39abce5c1a Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Mon, 13 Dec 2021 12:36:40 +0100 Subject: [PATCH 097/175] Pin agents 6.32.2 and 7.32.2 --- bin/compile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/compile b/bin/compile index 866b360..7d771ee 100644 --- a/bin/compile +++ b/bin/compile @@ -11,8 +11,8 @@ set -o pipefail # set -x # Set agent pinned version -DD_AGENT_PINNED_VERSION_6="6.31.0-1" -DD_AGENT_PINNED_VERSION_7="7.31.0-1" +DD_AGENT_PINNED_VERSION_6="6.32.2-1" +DD_AGENT_PINNED_VERSION_7="7.32.2-1" # Parse and derive params BUILD_DIR=$1 From f8e403f802d390fd0b02040e8abdfcbac9f56220 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Mon, 13 Dec 2021 15:54:51 +0100 Subject: [PATCH 098/175] Prepare release 1.30 --- CHANGELOG.md | 11 +++++++++++ extra/datadog.sh | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d3020d6..7d48519 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,17 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [Unreleased & outstanding issues] +- The `process-agent` increased in size between 7.31 and 7.32. We are currently investigating the reason. We are aware of the slug size issues with this buildpack and we are working on providing a long term solution. For updates on this you can follow [issue 269 in Github](https://github.com/DataDog/heroku-buildpack-datadog/issues/269) + +## [1.30] - 2021-12-13 + +### Security +- Agents `6.32.2` and `7.32.2` set `-Dlog4j2.formatMsgNoLookups=True` when starting the JMXfetch process to mitigate the vulnerability described in [CVE-2021-44228](https://nvd.nist.gov/vuln/detail/CVE-2021-44228) + +### Added +- Datadog agent pinned versions are now `6.32.2` and `7.32.2` + ## [1.29] - 2021-09-23 ### Added diff --git a/extra/datadog.sh b/extra/datadog.sh index d0219f5..0bb8e90 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -50,7 +50,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="dev" +BUILDPACKVERSION="1.30" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # We want always to have the Dyno ID as a host alias to improve correlation From 30c642aa7c31fbf3ee9587f063fea7060d377720 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Mon, 13 Dec 2021 11:34:39 -0500 Subject: [PATCH 099/175] update wording --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d48519..570c59d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). ## [Unreleased & outstanding issues] -- The `process-agent` increased in size between 7.31 and 7.32. We are currently investigating the reason. We are aware of the slug size issues with this buildpack and we are working on providing a long term solution. For updates on this you can follow [issue 269 in Github](https://github.com/DataDog/heroku-buildpack-datadog/issues/269) +- The `process-agent` increased in size between 7.31 and 7.32. We are aware the increasing slug size with this buildpack presents issues for some users, and we are working on providing a long term solution. For updates on this you can follow [issue 269 in Github](https://github.com/DataDog/heroku-buildpack-datadog/issues/269) ## [1.30] - 2021-12-13 From f54ef45dd9aa269457b66354aa01a9101b41cd67 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Mon, 13 Dec 2021 17:37:53 +0100 Subject: [PATCH 100/175] Move buildpack version back to dev --- extra/datadog.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index 0bb8e90..d0219f5 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -50,7 +50,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="1.30" +BUILDPACKVERSION="dev" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # We want always to have the Dyno ID as a host alias to improve correlation From 4b8563f834dcb5b05c4c519038b5ac88180d8406 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Thu, 16 Dec 2021 11:21:39 +0100 Subject: [PATCH 101/175] Pin agent versions 6.32.3 and 7.32.3 --- bin/compile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/compile b/bin/compile index 7d771ee..35ae4be 100644 --- a/bin/compile +++ b/bin/compile @@ -11,8 +11,8 @@ set -o pipefail # set -x # Set agent pinned version -DD_AGENT_PINNED_VERSION_6="6.32.2-1" -DD_AGENT_PINNED_VERSION_7="7.32.2-1" +DD_AGENT_PINNED_VERSION_6="6.32.3-1" +DD_AGENT_PINNED_VERSION_7="7.32.3-1" # Parse and derive params BUILD_DIR=$1 From 6e1c1603a02ca0dc2392b4830b4e6781b5c50692 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Thu, 16 Dec 2021 11:42:19 +0100 Subject: [PATCH 102/175] Prepare release 1.31 --- CHANGELOG.md | 9 +++++++++ extra/datadog.sh | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 570c59d..082a1f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased & outstanding issues] - The `process-agent` increased in size between 7.31 and 7.32. We are aware the increasing slug size with this buildpack presents issues for some users, and we are working on providing a long term solution. For updates on this you can follow [issue 269 in Github](https://github.com/DataDog/heroku-buildpack-datadog/issues/269) +## [1.31] - 2021-12-16 + +### Security + +- Agents `6.32.3` and `7.33.3` upgrade the log4j dependency to 2.12.2 in JMXFetch to fully address [CVE-2021-44228](https://nvd.nist.gov/vuln/detail/CVE-2021-44228) and [CVE-2021-45046](https://nvd.nist.gov/vuln/detail/CVE-2021-45046) + +### Added +- Datadog agent pinned versions are now `6.32.3` and `7.32.3` + ## [1.30] - 2021-12-13 ### Security diff --git a/extra/datadog.sh b/extra/datadog.sh index d0219f5..080e7ea 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -50,7 +50,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="dev" +BUILDPACKVERSION="1.31" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # We want always to have the Dyno ID as a host alias to improve correlation From 92440cf38abda8223ce0104c8fa748dc5516497a Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Thu, 16 Dec 2021 15:53:20 +0100 Subject: [PATCH 103/175] Revert buildpack version back to dev --- extra/datadog.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index 080e7ea..d0219f5 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -50,7 +50,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="1.31" +BUILDPACKVERSION="dev" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # We want always to have the Dyno ID as a host alias to improve correlation From 8a5c7c2eb6fa19f236c4e8dd8bd96628874d8a98 Mon Sep 17 00:00:00 2001 From: Olivier Vielpeau Date: Fri, 17 Dec 2021 21:36:03 +0100 Subject: [PATCH 104/175] [readme] Recommend Agent v7 --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f826728..814fe1e 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,9 @@ export APPNAME= ```shell cd +# Use the latest major Agent version +export DD_AGENT_MAJOR_VERSION=7 + # Enable Heroku Labs Dyno Metadata to set HEROKU_APP_NAME env variable automatically heroku labs:enable runtime-dyno-metadata -a $APPNAME @@ -105,7 +108,7 @@ In addition to the environment variables shown above, there are several others y | `DD_PROCESS_AGENT` | *Optional.* The Datadog Process Agent is disabled by default. Set this to `true` to enable the Process Agent. Changing this option requires recompilation of the slug. Check [the upgrading and slug recompilation section](#upgrading-and-slug-recompilation) for details. | | `DD_SITE` | *Optional.* If you use the app.datadoghq.eu service, set this to `datadoghq.eu`. Defaults to `datadoghq.com`. | | `DD_AGENT_VERSION` | *Optional.* By default, the buildpack installs the latest 6.x version of the Datadog Agent available in the package repository. Use this variable to install older versions of the Datadog Agent. **Note**: Not all versions of the Agent may be available. This option takes precedence over `DD_AGENT_MAJOR_VERSION`. Changing this option requires recompiling the slug. See [upgrading and slug recompilation](#upgrading-and-slug-recompilation) for details. | -| `DD_AGENT_MAJOR_VERSION` | *Optional.* By default, the buildpack installs the latest 6.x version of the Datadog Agent available in the package repository. Set this variable to `7` to install the latest 7.x version of the Datadog Agent. Check the [Python versions section](#python-and-agent-versions) for more information on the relation of the Agent version and Python version. Changing this option requires recompiling the slug. See [upgrading and slug recompilation](#upgrading-and-slug-recompilation) for details. | +| `DD_AGENT_MAJOR_VERSION` | *Optional.* By default, the buildpack installs the latest 6.x version of the Datadog Agent available in the package repository. Set this variable to `7` to install the latest 7.x version of the Datadog Agent. **Setting this variable to `7` is highly recommended**. Check the [Python versions section](#python-and-agent-versions) for more information on the relation of the Agent version and Python version. Changing this option requires recompiling the slug. See [upgrading and slug recompilation](#upgrading-and-slug-recompilation) for details. | | `DD_DISABLE_HOST_METRICS` | *Optional.* By default, the buildpack reports system metrics for the host machine running the dyno. Set this to `true` to disable system metrics collection. See the [system metrics section](#system-metrics) below for more information. | | `DD_PYTHON_VERSION` | *Optional.* Starting with version `6.14.0`, Datadog Agent ships with Python versions `2` and `3`. The buildpack only keeps one of the versions. Set this to `2` or `3` to select the Python version you want the Agent to keep. If not set, the buildpack keeps `2`. Check the [Python versions section](#python-and-agent-versions) for more information. Changing this option requires recompiling the slug. Check [the upgrading and slug recompilation section](#upgrading-and-slug-recompilation) for details. | | `DD_HEROKU_CONF_FOLDER` | *Optional.* By default, the buildpack looks in the root of your application for a folder `/datadog` for any configuration files you wish to include, see [prerun.sh script](#prerun-script). This location can be overridden by setting this to your desired path. | From 7f2219445e93e3b64835073ba90efbb77e7ae88b Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Tue, 21 Dec 2021 15:05:06 +0100 Subject: [PATCH 105/175] Update README Co-authored-by: Ethan Lowman <53835328+ethan-lowman-dd@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 814fe1e..98372ba 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ export APPNAME= cd # Use the latest major Agent version -export DD_AGENT_MAJOR_VERSION=7 +heroku config:add DD_AGENT_MAJOR_VERSION=7 # Enable Heroku Labs Dyno Metadata to set HEROKU_APP_NAME env variable automatically heroku labs:enable runtime-dyno-metadata -a $APPNAME From 0fd9cdf0f24d168fcc01d934183ddf6ef7120803 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Thu, 23 Dec 2021 16:07:13 -0500 Subject: [PATCH 106/175] Prepare release 1.32 --- CHANGELOG.md | 9 +++++++++ extra/datadog.sh | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 082a1f3..54c58db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased & outstanding issues] - The `process-agent` increased in size between 7.31 and 7.32. We are aware the increasing slug size with this buildpack presents issues for some users, and we are working on providing a long term solution. For updates on this you can follow [issue 269 in Github](https://github.com/DataDog/heroku-buildpack-datadog/issues/269) +## [1.32] - 2021-12-23 + +### Security + +- Agents `6.32.4` and `7.33.4` remove all dependencies on log4j and use java.util.logging instead. + +### Added +- Datadog agent pinned versions are now `6.32.4` and `7.32.4` + ## [1.31] - 2021-12-16 ### Security diff --git a/extra/datadog.sh b/extra/datadog.sh index d0219f5..719e58c 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -50,7 +50,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="dev" +BUILDPACKVERSION="1.32" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # We want always to have the Dyno ID as a host alias to improve correlation From 817663bb6566610a1962c53bd471716d7799cdda Mon Sep 17 00:00:00 2001 From: Jeremy Date: Thu, 23 Dec 2021 16:10:17 -0500 Subject: [PATCH 107/175] Pin agent versions 6.32.4 and 7.32.4 --- bin/compile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/compile b/bin/compile index 35ae4be..76404de 100644 --- a/bin/compile +++ b/bin/compile @@ -11,8 +11,8 @@ set -o pipefail # set -x # Set agent pinned version -DD_AGENT_PINNED_VERSION_6="6.32.3-1" -DD_AGENT_PINNED_VERSION_7="7.32.3-1" +DD_AGENT_PINNED_VERSION_6="6.32.4-1" +DD_AGENT_PINNED_VERSION_7="7.32.4-1" # Parse and derive params BUILD_DIR=$1 From fbbf7567aae3b2d0b9caa3ad50b9bbf833548645 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Mon, 27 Dec 2021 16:57:30 +0100 Subject: [PATCH 108/175] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 54c58db..d7bc686 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased & outstanding issues] - The `process-agent` increased in size between 7.31 and 7.32. We are aware the increasing slug size with this buildpack presents issues for some users, and we are working on providing a long term solution. For updates on this you can follow [issue 269 in Github](https://github.com/DataDog/heroku-buildpack-datadog/issues/269) -## [1.32] - 2021-12-23 +## [1.32] - 2021-12-27 ### Security From 3938149dede0f3a8e65eca16f354448a0bdcf191 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Mon, 27 Dec 2021 16:59:18 +0100 Subject: [PATCH 109/175] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7bc686..b17a734 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Security -- Agents `6.32.4` and `7.33.4` remove all dependencies on log4j and use java.util.logging instead. +- Agents `6.32.4` and `7.32.4` remove all dependencies on log4j and use java.util.logging instead. ### Added - Datadog agent pinned versions are now `6.32.4` and `7.32.4` From e2f05271325214a6b914b7c91cb6606996922a11 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Mon, 27 Dec 2021 17:13:48 +0100 Subject: [PATCH 110/175] Revert buildpack version back to dev --- extra/datadog.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index 719e58c..d0219f5 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -50,7 +50,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="1.32" +BUILDPACKVERSION="dev" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # We want always to have the Dyno ID as a host alias to improve correlation From f3e83d4c3f00216aa25598636ff34ab2ec1d52c9 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 26 Jan 2022 15:03:07 +0100 Subject: [PATCH 111/175] Ability to add custom checks --- extra/datadog.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/extra/datadog.sh b/extra/datadog.sh index d0219f5..e0a1415 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -38,7 +38,9 @@ sed -i -e"s|^.*cloud_provider_metadata:.*$|cloud_provider_metadata: []|" "$DATAD APP_DATADOG_DEFAULT="/app/datadog" APP_DATADOG="${DD_HEROKU_CONF_FOLDER:=$APP_DATADOG_DEFAULT}" APP_DATADOG_CONF_DIR="$APP_DATADOG/conf.d" +APP_DATADOG_CHECKS_DIR="$APP_DATADOG/checks.d" +# Agent integrations configuration for file in "$APP_DATADOG_CONF_DIR"/*.yaml; do test -e "$file" || continue # avoid errors when glob doesn't match anything filename="$(basename -- "$file")" @@ -47,6 +49,18 @@ for file in "$APP_DATADOG_CONF_DIR"/*.yaml; do cp "$file" "$DD_CONF_DIR/conf.d/${filename}.d/conf.yaml" done +# Custom checks configuration +for file in "$APP_DATADOG_CHECKS_DIR"/*.yaml; do + test -e "$file" || continue # avoid errors when glob doesn't match anything + cp "$file" "$DD_CONF_DIR/conf.d/" +done + +# Custom checks code +for file in "$APP_DATADOG_CHECKS_DIR"/*.py; do + test -e "$file" || continue # avoid errors when glob doesn't match anything + cp "$file" "$DD_CONF_DIR/checks.d/" +done + # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} From 1079a8b7b9a84c133c1db2e9782fe7715f6e3b47 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 26 Jan 2022 15:48:33 +0100 Subject: [PATCH 112/175] Add documentation about enabling custom checks --- README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/README.md b/README.md index 244dfc0..b45b973 100644 --- a/README.md +++ b/README.md @@ -188,6 +188,23 @@ For example, to install the [ping integration][21], create the configuration fil agent-wrapper integration install -t datadog-ping==1.0.0 ``` +## Enabling custom checks + +To enable your own [Agent Custom Checks][29], create a `checks.d` folder in the datadog configuration folder within your application. Under it, copy all `.py` and `.yaml` files from your custom checks. During the dyno start up, your files are copied to the appropriate Datadog Agent configuration directories. + +For example, if you have two custom checks, `foo` and `bar`, this would be the right folder tree: + +``` +. +└── app + └── datadog + └── checks.d + ├── foo.py + ├── foo.yaml + ├── bar.py + └── bar.yaml +``` + ## Prerun script In addition to all of the configurations above, you can include a prerun script, `/datadog/prerun.sh`, in your application. The prerun script runs after all of the standard configuration actions and immediately before starting the Datadog Agent. This allows you to modify the environment variables (for example: DD_TAGS or DD_VERSION), perform additional configurations, install community integrations, or even disable the Datadog Agent programmatically. @@ -465,3 +482,4 @@ After an upgrade of the buildpack or Agent, you must recompile your application' [26]: https://github.com/DataDog/heroku-buildpack-datadog [27]: https://github.com/miketheman/heroku-buildpack-datadog [28]: https://github.com/DataDog/heroku-buildpack-datadog/blob/master/CHANGELOG.md +[29]: https://docs.datadoghq.com/developers/custom_checks/ From 6cadb54b2cfd606cef79194aca863f43c8d6ad5c Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 26 Jan 2022 15:51:56 +0100 Subject: [PATCH 113/175] Reorganize doc links --- README.md | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index b45b973..15e8bd6 100644 --- a/README.md +++ b/README.md @@ -190,7 +190,7 @@ agent-wrapper integration install -t datadog-ping==1.0.0 ## Enabling custom checks -To enable your own [Agent Custom Checks][29], create a `checks.d` folder in the datadog configuration folder within your application. Under it, copy all `.py` and `.yaml` files from your custom checks. During the dyno start up, your files are copied to the appropriate Datadog Agent configuration directories. +To enable your own [Agent Custom Checks][22], create a `checks.d` folder in the datadog configuration folder within your application. Under it, copy all `.py` and `.yaml` files from your custom checks. During the dyno start up, your files are copied to the appropriate Datadog Agent configuration directories. For example, if you have two custom checks, `foo` and `bar`, this would be the right folder tree: @@ -258,7 +258,7 @@ To reduce your slug size, make sure that `DD_APM_ENABLED` is set to `false`, if ## Debugging -To run any of the [information or debugging commands][22], use the `agent-wrapper` command. +To run any of the [information or debugging commands][23], use the `agent-wrapper` command. For example, to display the status of your Datadog Agent and enabled integrations, run: @@ -278,7 +278,7 @@ The Datadog buildpack does not collect logs from the Heroku platform. To set up ## Using Heroku with Docker images -This buildpack only works for Heroku deployments that use [Heroku's Slug Compiler][23]. If you are deploying your application in Heroku using Docker containers, add the Datadog Agent as part of your Docker image and start the Agent as a different process in your container. +This buildpack only works for Heroku deployments that use [Heroku's Slug Compiler][24]. If you are deploying your application in Heroku using Docker containers, add the Datadog Agent as part of your Docker image and start the Agent as a different process in your container. As an example, if you are building your Docker image using a Debian based OS, add the following lines to your `Dockerfile`: @@ -325,15 +325,15 @@ datadog-agent run & /opt/datadog-agent/embedded/bin/process-agent --config=/etc/datadog-agent/datadog.yaml ``` -For more advanced options in the Docker image, reference the [Datadog Agent Docker files][24]. +For more advanced options in the Docker image, reference the [Datadog Agent Docker files][25]. ## Contributing -See the [contributing guidelines][25] to learn how to open an issue or PR to the [Heroku-buildpack-datadog repository][26]. +See the [contributing guidelines][26] to learn how to open an issue or PR to the [Heroku-buildpack-datadog repository][27]. ## History -Earlier versions of this project were forked from the [miketheman heroku-buildpack-datadog project][27]. It was largely rewritten for Datadog's Agent version 6. Changes and more information can be found in the [changelog][28]. +Earlier versions of this project were forked from the [miketheman heroku-buildpack-datadog project][28]. It was largely rewritten for Datadog's Agent version 6. Changes and more information can be found in the [changelog][29]. ## Troubleshooting @@ -475,11 +475,11 @@ After an upgrade of the buildpack or Agent, you must recompile your application' [19]: https://github.com/DataDog/integrations-core/blob/master/redisdb/datadog_checks/redisdb/data/conf.yaml.example [20]: https://github.com/DataDog/integrations-extras/ [21]: https://github.com/DataDog/integrations-extras/tree/master/ping -[22]: https://docs.datadoghq.com/agent/guide/agent-commands/#agent-status-and-information -[23]: https://devcenter.heroku.com/articles/slug-compiler -[24]: https://github.com/DataDog/datadog-agent/tree/master/Dockerfiles -[25]: https://github.com/DataDog/heroku-buildpack-datadog/blob/master/CONTRIBUTING.md -[26]: https://github.com/DataDog/heroku-buildpack-datadog -[27]: https://github.com/miketheman/heroku-buildpack-datadog -[28]: https://github.com/DataDog/heroku-buildpack-datadog/blob/master/CHANGELOG.md -[29]: https://docs.datadoghq.com/developers/custom_checks/ +[22]: https://docs.datadoghq.com/developers/custom_checks/ +[23]: https://docs.datadoghq.com/agent/guide/agent-commands/#agent-status-and-information +[24]: https://devcenter.heroku.com/articles/slug-compiler +[25]: https://github.com/DataDog/datadog-agent/tree/master/Dockerfiles +[26]: https://github.com/DataDog/heroku-buildpack-datadog/blob/master/CONTRIBUTING.md +[27]: https://github.com/DataDog/heroku-buildpack-datadog +[28]: https://github.com/miketheman/heroku-buildpack-datadog +[29]: https://github.com/DataDog/heroku-buildpack-datadog/blob/master/CHANGELOG.md From b606706313017fb4bf8210890f374f2aebd5d5af Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 26 Jan 2022 16:15:23 +0100 Subject: [PATCH 114/175] Add documentation on disabling integrations based on dyno --- README.md | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 15e8bd6..8e8481a 100644 --- a/README.md +++ b/README.md @@ -176,6 +176,8 @@ instances: **Note**: See the sample [redisdb.d/conf.yaml][19] for all available configuration options. +### Community Integrations + If the integration you are enabling is part of the [Community Integrations][20], install the package as part of the [prerun script](#prerun-script). ``` @@ -188,6 +190,18 @@ For example, to install the [ping integration][21], create the configuration fil agent-wrapper integration install -t datadog-ping==1.0.0 ``` +### Disabling integrations based on dynos + +As the filesystem in a Heroku application will be shared by all dynos, if you enable an integration, it will be run on every dyno, including `run` or `worker` dynos. If you want to limit the integration runs based on dyno name or type, you can do that adding a small snippet to the [prerun script](#prerun-script). + +For example, if the Gunicorn only needs to be run on `web` type dynos, add the following to your prerun script: + +``` +if [ "$DYNOTYPE" != "web" ]; then + rm -f "$DD_CONF_DIR/conf.d/gunicorn.d/conf.yaml" +fi +``` + ## Enabling custom checks To enable your own [Agent Custom Checks][22], create a `checks.d` folder in the datadog configuration folder within your application. Under it, copy all `.py` and `.yaml` files from your custom checks. During the dyno start up, your files are copied to the appropriate Datadog Agent configuration directories. @@ -203,7 +217,6 @@ For example, if you have two custom checks, `foo` and `bar`, this would be the r ├── foo.yaml ├── bar.py └── bar.yaml -``` ## Prerun script @@ -219,6 +232,11 @@ if [ "$DYNOTYPE" == "run" ]; then DISABLE_DATADOG_AGENT="true" fi +# Disable integrations based on dyno type +if [ "$DYNOTYPE" != "web" ]; then + rm -f "$DD_CONF_DIR/conf.d/gunicorn.d/conf.yaml" +fi + # Set app version based on HEROKU_SLUG_COMMIT if [ -n "$HEROKU_SLUG_COMMIT" ]; then DD_VERSION=$HEROKU_SLUG_COMMIT From 816e57866c121080dcd1feb96613dd5bb5073c11 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 26 Jan 2022 16:18:00 +0100 Subject: [PATCH 115/175] Fix formatting --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 8e8481a..969f080 100644 --- a/README.md +++ b/README.md @@ -218,6 +218,8 @@ For example, if you have two custom checks, `foo` and `bar`, this would be the r ├── bar.py └── bar.yaml +``` + ## Prerun script In addition to all of the configurations above, you can include a prerun script, `/datadog/prerun.sh`, in your application. The prerun script runs after all of the standard configuration actions and immediately before starting the Datadog Agent. This allows you to modify the environment variables (for example: DD_TAGS or DD_VERSION), perform additional configurations, install community integrations, or even disable the Datadog Agent programmatically. From 3ba2a26885dc2ca20f9bf8175a133378e76c47e0 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 26 Jan 2022 16:26:15 +0100 Subject: [PATCH 116/175] Apply suggestions from review Co-authored-by: Jeremy --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 969f080..ca565f8 100644 --- a/README.md +++ b/README.md @@ -194,7 +194,7 @@ agent-wrapper integration install -t datadog-ping==1.0.0 As the filesystem in a Heroku application will be shared by all dynos, if you enable an integration, it will be run on every dyno, including `run` or `worker` dynos. If you want to limit the integration runs based on dyno name or type, you can do that adding a small snippet to the [prerun script](#prerun-script). -For example, if the Gunicorn only needs to be run on `web` type dynos, add the following to your prerun script: +For example, if the Gunicorn integration only needs to run on `web` type dynos, add the following to your prerun script: ``` if [ "$DYNOTYPE" != "web" ]; then From 19ceeb8ad680745d1c38f37dcc3abf4165a3bc33 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Thu, 27 Jan 2022 13:09:05 +0100 Subject: [PATCH 117/175] Ping 6.33 and 7.33 --- bin/compile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/compile b/bin/compile index 76404de..01800de 100644 --- a/bin/compile +++ b/bin/compile @@ -11,8 +11,8 @@ set -o pipefail # set -x # Set agent pinned version -DD_AGENT_PINNED_VERSION_6="6.32.4-1" -DD_AGENT_PINNED_VERSION_7="7.32.4-1" +DD_AGENT_PINNED_VERSION_6="6.33.0-1" +DD_AGENT_PINNED_VERSION_7="7.33.0-1" # Parse and derive params BUILD_DIR=$1 From f67f9101999ccc008d3d65c850b48190ce92087b Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Thu, 27 Jan 2022 14:17:39 +0100 Subject: [PATCH 118/175] Prepare release 1.33 --- CHANGELOG.md | 7 ++++++- extra/datadog.sh | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b17a734..7a8caa3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,10 +7,15 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased & outstanding issues] - The `process-agent` increased in size between 7.31 and 7.32. We are aware the increasing slug size with this buildpack presents issues for some users, and we are working on providing a long term solution. For updates on this you can follow [issue 269 in Github](https://github.com/DataDog/heroku-buildpack-datadog/issues/269) +## [1.33] - 2022-01-27 + +### Added +- Datadog agent pinned versions are now `6.33.0` and `7.33.0` +- Custom checks are now easier to set up. (https://docs.datadoghq.com/agent/basic_agent_usage/heroku/#enabling-custom-checks) + ## [1.32] - 2021-12-27 ### Security - - Agents `6.32.4` and `7.32.4` remove all dependencies on log4j and use java.util.logging instead. ### Added diff --git a/extra/datadog.sh b/extra/datadog.sh index e0a1415..d85063b 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -64,7 +64,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="dev" +BUILDPACKVERSION="1.33" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # We want always to have the Dyno ID as a host alias to improve correlation From 19e88059788784735d94a79ead6b33dddfb4da56 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Thu, 27 Jan 2022 16:22:50 +0100 Subject: [PATCH 119/175] Revert buildpack version back to dev --- extra/datadog.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index d85063b..e0a1415 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -64,7 +64,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="1.33" +BUILDPACKVERSION="dev" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # We want always to have the Dyno ID as a host alias to improve correlation From 1f9efde59add6eaef92bc2c400cef9d3dd2dc24c Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Tue, 15 Feb 2022 10:02:56 +0100 Subject: [PATCH 120/175] Fix hardcoded python symlinks --- extra/datadog.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index e0a1415..f66e737 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -139,12 +139,12 @@ if [ "$DD_PYTHON_VERSION" = "3" ]; then if [ "$DD_AGENT_VERSION" != "$(echo -e "$DD_AGENT_BASE_VERSION\n$DD_AGENT_VERSION" | sort -V | head -n1)" ]; then # If Python version is 3, it has to be specified in the configuration file echo 'python_version: 3' >> $DATADOG_CONF + fi # Update symlinks to Python binaries ln -sfn "$DD_DIR"/embedded/bin/python3 "$DD_DIR"/embedded/bin/python ln -sfn "$DD_DIR"/embedded/bin/python3-config "$DD_DIR"/embedded/bin/python-config ln -sfn "$DD_DIR"/embedded/bin/pip3 "$DD_DIR"/embedded/bin/pip ln -sfn "$DD_DIR"/embedded/bin/pydoc3 "$DD_DIR"/embedded/bin/pydoc - fi fi # Ensure all check and library locations are findable in the Python path. From 3996cd3501d507ad086ecb7195e4c04d67b947e0 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Mon, 21 Feb 2022 13:10:51 +0100 Subject: [PATCH 121/175] Remove unneeded files --- bin/compile | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/bin/compile b/bin/compile index 01800de..cd5ded7 100644 --- a/bin/compile +++ b/bin/compile @@ -172,11 +172,19 @@ dpkg -x "$DEB" "$APT_DIR" # Removing unneeded content rm -rf "$APT_DIR"/opt/datadog-agent/sources \ "$APT_DIR"/opt/datadog-agent/embedded/share/doc \ - "$APT_DIR"/opt/datadog-agent/embedded/share/man + "$APT_DIR"/opt/datadog-agent/embedded/share/man \ + "$APT_DIR"/opt/datadog-agent/embedded/.installed_by_pkg.txt # Remove Cryptodome Selftest rm -rf "$APT_DIR"/opt/datadog-agent/embedded/lib/python*/site-packages/Cryptodome/SelfTest +# Remove unused botocore data +for i in `ls $APT_DIR/opt/datadog-agent/embedded/lib/python*/site-packages/botocore/data`; do + if [ "$i" != "s3" ]; then + rm -rf "$APT_DIR"/opt/datadog-agent/embedded/lib/python*/site-packages/botocore/data/"$i" + fi +done + # Remove unnedded Python packages for i in "${DELETE_PACKAGES[@]}"; do rm -rf "$APT_DIR"/opt/datadog-agent/embedded/lib/python*/site-packages/"$i" @@ -185,9 +193,11 @@ done # Remove the system-probe binary, only needed for network performance monitoring rm -rf "$APT_DIR/opt/datadog-agent/embedded/bin/system-probe" rm -rf "$APT_DIR"/opt/datadog-agent/embedded/lib/libbcc* +rm -rf "$APT_DIR"/opt/datadog-agent/embedded/nikos/ # Remove static libraries rm -rf "$APT_DIR"/opt/datadog-agent/embedded/lib/*.a +rm -rf "$APT_DIR"/opt/datadog-agent/embedded/lib/python*/config*/*.a # Remove the Security agent rm -rf "$APT_DIR/opt/datadog-agent/embedded/bin/security-agent" From 2d5bab968d281ca5bff98caf95208932d78c8cc3 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Mon, 21 Feb 2022 14:41:44 +0100 Subject: [PATCH 122/175] For newer agent versions, get the custom Heroku build --- bin/compile | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/bin/compile b/bin/compile index 01800de..82c0dde 100644 --- a/bin/compile +++ b/bin/compile @@ -142,22 +142,32 @@ fi AGENT_VERSIONS=$(apt-cache $APT_OPTIONS show datadog-agent | grep "Version: ") AGENT_VERSIONS=$(sed 's/Version: 1://g' <<<"$AGENT_VERSIONS") - -# If specified version doesn't exist, list available versions. -if [ -z $(echo "$AGENT_VERSIONS" | grep -x "$DD_AGENT_VERSION") ]; then - topic "ERROR: Version \"$DD_AGENT_VERSION\" was not found." - echo "Available versions:" | indent - echo "$AGENT_VERSIONS" | indent - exit 1 +AGENT_HEROKU_VERSIONS=$(apt-cache $APT_OPTIONS show datadog-heroku-agent | grep "Version: ") +AGENT_HEROKU_VERSIONS=$(sed 's/Version: 1://g' <<<"$AGENT_HEROKU_VERSIONS") + +PACKAGE_NAME="datadog-heroku-agent" + +# If specified version doesn't exist as heroku build, try regular agent +if [ -z $(echo "$AGENT_HEROKU_VERSIONS" | grep -x "$DD_AGENT_VERSION") ]; then + # If specified version doesn't exist as regular build either, err and quit + if [ -z $(echo "$AGENT_VERSIONS" | grep -x "$DD_AGENT_VERSION") ]; then + topic "ERROR: Version \"$DD_AGENT_VERSION\" was not found." + echo "Available Heroku versions:" | indent + echo "$AGENT_HEROKU_VERSIONS" | indent + echo "Available regular Agent versions:" | indent + echo "$AGENT_VERSIONS" | indent + exit 1 + fi + PACKAGE_NAME="datadog-agent" fi # Set the specified version. -PACKAGE="datadog-agent=1:$DD_AGENT_VERSION" +PACKAGE="$PACKAGE_NAME=1:$DD_AGENT_VERSION" topic "Downloading Datadog Agent $DD_AGENT_VERSION" apt-get $APT_OPTIONS -y --force-yes -d install --reinstall --no-install-recommends "$PACKAGE" | indent -DPKG_STUB="$APT_CACHE_DIR/archives/datadog-agent_1%3a" +DPKG_STUB="$APT_CACHE_DIR/archives/${PACKAGE_NAME}_1%3a" if [ -z $DD_AGENT_VERSION ]; then DEB=$(ls -t "$DPKG_STUB"*.deb | head -n 1) DD_AGENT_VERSION=${DEB:${#DPKG_STUB}:(${#DEB}-${#DPKG_STUB}-10)} From 2dd11bf9d8cfd82d8b8ccb03cf811ddfd175bc8d Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 2 Mar 2022 12:15:52 +0100 Subject: [PATCH 123/175] Pin 7.34 and 6.34 as default versions --- bin/compile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/compile b/bin/compile index 82c0dde..ca6a673 100644 --- a/bin/compile +++ b/bin/compile @@ -11,8 +11,8 @@ set -o pipefail # set -x # Set agent pinned version -DD_AGENT_PINNED_VERSION_6="6.33.0-1" -DD_AGENT_PINNED_VERSION_7="7.33.0-1" +DD_AGENT_PINNED_VERSION_6="6.34.0-1" +DD_AGENT_PINNED_VERSION_7="7.34.0-1" # Parse and derive params BUILD_DIR=$1 From d0d1873dd1ef5179c554aa02f1d1303191569d80 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 2 Mar 2022 15:04:55 +0100 Subject: [PATCH 124/175] Prepare release 2.0 of the buildpack --- CHANGELOG.md | 9 +++++++++ extra/datadog.sh | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a8caa3..7cb4c1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased & outstanding issues] - The `process-agent` increased in size between 7.31 and 7.32. We are aware the increasing slug size with this buildpack presents issues for some users, and we are working on providing a long term solution. For updates on this you can follow [issue 269 in Github](https://github.com/DataDog/heroku-buildpack-datadog/issues/269) +## [2.0] - 2022-03-02 + +### Added +- Specific Heroku Datadog Agent build (starting with `6.34.0` and `7.34.0`) is now used +- Datadog agent pinned versions are now `6.34.0` and `7.34.0` + +### Fixed +- Community integrations can now be installed properly when the Python versions for the agent and the system are different + ## [1.33] - 2022-01-27 ### Added diff --git a/extra/datadog.sh b/extra/datadog.sh index e0a1415..c33777c 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -64,7 +64,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="dev" +BUILDPACKVERSION="2.0" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # We want always to have the Dyno ID as a host alias to improve correlation From df5a641954ab3a3cd3a04a551ebebc059c214fe5 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 2 Mar 2022 15:50:23 +0100 Subject: [PATCH 125/175] Move buildpack version back to dev --- extra/datadog.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index a544332..f66e737 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -64,7 +64,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="2.0" +BUILDPACKVERSION="dev" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # We want always to have the Dyno ID as a host alias to improve correlation From 8906c88901baa964215175c55b8dece67c58fa27 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Thu, 3 Mar 2022 11:37:05 +0100 Subject: [PATCH 126/175] prepare 2.0 release --- extra/datadog.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index f66e737..a544332 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -64,7 +64,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="dev" +BUILDPACKVERSION="2.0" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # We want always to have the Dyno ID as a host alias to improve correlation From 18fc145e2bbde510fd6a48e94c9a8281239ab790 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Thu, 3 Mar 2022 11:41:25 +0100 Subject: [PATCH 127/175] Revert buildpack version back to dev --- extra/datadog.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index a544332..f66e737 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -64,7 +64,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="2.0" +BUILDPACKVERSION="dev" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # We want always to have the Dyno ID as a host alias to improve correlation From f24779bc4804fb5102cac75fc95b8206acc85ef5 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 30 Mar 2022 15:18:06 +0200 Subject: [PATCH 128/175] Only test for size starting on agent 7.34/6.34 --- test/compile_and_run_test.sh | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/test/compile_and_run_test.sh b/test/compile_and_run_test.sh index 9f55fcd..daa51cb 100644 --- a/test/compile_and_run_test.sh +++ b/test/compile_and_run_test.sh @@ -46,13 +46,11 @@ compileAndRunVersion() SIZE_LIB=$(du -s $HOME/.apt/opt/datadog-agent/embedded/lib | cut -f1) SIZE_EMB_BIN=$(du -s $HOME/.apt/opt/datadog-agent/embedded/bin | cut -f1) - # Prior to 6.13 the agent was too big - BIGSIZE_VERSION="6.13.0-1" - if test "$BIGSIZE_VERSION" = "$(echo "$BIGSIZE_VERSION\n$1" | sort -V | head -n1)" ; then - assertTrue "Binary folder is too big: ${SIZE_BIN}" "[ $SIZE_BIN -lt 105000 ]" + if $2; then + assertTrue "Binary folder is too big: ${SIZE_BIN}" "[ $SIZE_BIN -lt 80000 ]" + assertTrue "Embedded library folder is too big: ${SIZE_LIB}" "[ $SIZE_LIB -lt 180000 ]" + assertTrue "Embedded binary folder is too big: ${SIZE_EMB_BIN}" "[ $SIZE_EMB_BIN -lt 80000 ]" fi - assertTrue "Embedded library folder is too big: ${SIZE_LIB}" "[ $SIZE_LIB -lt 255000 ]" - assertTrue "Embedded binary folder is too big: ${SIZE_EMB_BIN}" "[ $SIZE_EMB_BIN -lt 115000 ]" rm -rf $HOME/.apt/ } @@ -64,7 +62,12 @@ testReleased6Versions() echo "deb [signed-by=SIGNED_BY_PLACEHOLDER] https://apt.datadoghq.com/ stable 6" > "${BUILDPACK_HOME}/etc/datadog.list" for VERSION in ${AGENT_VERSIONS}; do - compileAndRunVersion $VERSION + # Only test for size from 6.34 onwards + if test "6.34.0-1" = "$(echo "6.34.0-1\n$VERSION" | sort -V | head -n1)" ; then + compileAndRunVersion $VERSION true + else + compileAndRunVersion $VERSION false + fi done } @@ -75,7 +78,12 @@ testReleased7Versions() echo "deb [signed-by=SIGNED_BY_PLACEHOLDER] https://apt.datadoghq.com/ stable 7" > "${BUILDPACK_HOME}/etc/datadog7.list" for VERSION in ${AGENT_VERSIONS}; do - compileAndRunVersion $VERSION + # Only test for size from 7.34 onwards + if test "7.34.0-1" = "$(echo "7.34.0-1\n$VERSION" | sort -V | head -n1)" ; then + compileAndRunVersion $VERSION true + else + compileAndRunVersion $VERSION false + fi done } From 40799b4b790c6fc5684bd27c58fad22682d71c87 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 20 Apr 2022 16:27:11 +0200 Subject: [PATCH 129/175] Pin versions 6.35.1 and 7.35.1 Signed-off-by: Ara Pulido --- bin/compile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/compile b/bin/compile index b7ccfc4..d5d9781 100644 --- a/bin/compile +++ b/bin/compile @@ -11,8 +11,8 @@ set -o pipefail # set -x # Set agent pinned version -DD_AGENT_PINNED_VERSION_6="6.34.0-1" -DD_AGENT_PINNED_VERSION_7="7.34.0-1" +DD_AGENT_PINNED_VERSION_6="6.35.1-1" +DD_AGENT_PINNED_VERSION_7="7.35.1-1" # Parse and derive params BUILD_DIR=$1 From 0d16a2405c0d5b0467f1e1a359c4a9b1b42f07bc Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Mon, 25 Apr 2022 11:14:56 +0200 Subject: [PATCH 130/175] Prepare release 2.1 --- CHANGELOG.md | 5 +++++ extra/datadog.sh | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7cb4c1a..7dab548 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased & outstanding issues] - The `process-agent` increased in size between 7.31 and 7.32. We are aware the increasing slug size with this buildpack presents issues for some users, and we are working on providing a long term solution. For updates on this you can follow [issue 269 in Github](https://github.com/DataDog/heroku-buildpack-datadog/issues/269) +## [2.1] - 2022-04-25 + +### Added +- Datadog agent pinned versions are now `6.35.1` and `7.35.1` + ## [2.0] - 2022-03-02 ### Added diff --git a/extra/datadog.sh b/extra/datadog.sh index f66e737..f1f7225 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -64,7 +64,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="dev" +BUILDPACKVERSION="2.1" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # We want always to have the Dyno ID as a host alias to improve correlation From 2b235abe84c9fcce5e9fac84d15bc7b1218c219e Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Mon, 25 Apr 2022 11:25:45 +0200 Subject: [PATCH 131/175] Move buildpack version back to dev --- extra/datadog.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index f1f7225..f66e737 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -64,7 +64,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="2.1" +BUILDPACKVERSION="dev" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # We want always to have the Dyno ID as a host alias to improve correlation From ae8c1fdc84ae89dd996315f67244face21c2742f Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 25 May 2022 11:50:23 +0200 Subject: [PATCH 132/175] Pin versions 7.36 and 6.36 --- bin/compile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/compile b/bin/compile index d5d9781..ffc6a74 100644 --- a/bin/compile +++ b/bin/compile @@ -11,8 +11,8 @@ set -o pipefail # set -x # Set agent pinned version -DD_AGENT_PINNED_VERSION_6="6.35.1-1" -DD_AGENT_PINNED_VERSION_7="7.35.1-1" +DD_AGENT_PINNED_VERSION_6="6.36.0-1" +DD_AGENT_PINNED_VERSION_7="7.36.0-1" # Parse and derive params BUILD_DIR=$1 From adf5ed3092599c172373ae641df7af99ef4902c0 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 25 May 2022 12:09:04 +0200 Subject: [PATCH 133/175] Changelog changes --- CHANGELOG.md | 6 ++++-- extra/datadog.sh | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7dab548..530b7d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). -## [Unreleased & outstanding issues] -- The `process-agent` increased in size between 7.31 and 7.32. We are aware the increasing slug size with this buildpack presents issues for some users, and we are working on providing a long term solution. For updates on this you can follow [issue 269 in Github](https://github.com/DataDog/heroku-buildpack-datadog/issues/269) +## [2.2] - 2022-05-25 + +### Added +- Datadog agent pinned versions are now `6.36.0` and `7.36.0` ## [2.1] - 2022-04-25 diff --git a/extra/datadog.sh b/extra/datadog.sh index f66e737..a909fdd 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -64,7 +64,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="dev" +BUILDPACKVERSION="2.2" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # We want always to have the Dyno ID as a host alias to improve correlation From 1102bbf29fb9121773fa5423b8420edf7ed1c880 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 25 May 2022 18:01:25 +0200 Subject: [PATCH 134/175] Move buildpack version back to dev --- extra/datadog.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index a909fdd..f66e737 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -64,7 +64,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="2.2" +BUILDPACKVERSION="dev" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # We want always to have the Dyno ID as a host alias to improve correlation From 214a858c816cc1ee882158a96cda729d71a7d167 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Tue, 12 Jul 2022 15:20:48 +0200 Subject: [PATCH 135/175] Improve docker on Heroku documentation --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ca565f8..2255f3c 100644 --- a/README.md +++ b/README.md @@ -298,7 +298,14 @@ The Datadog buildpack does not collect logs from the Heroku platform. To set up ## Using Heroku with Docker images -This buildpack only works for Heroku deployments that use [Heroku's Slug Compiler][24]. If you are deploying your application in Heroku using Docker containers, add the Datadog Agent as part of your Docker image and start the Agent as a different process in your container. +This buildpack only works for Heroku deployments that use [Heroku's Slug Compiler][24]. If you are deploying your application in Heroku using Docker containers: + +1. Add the Datadog Agent as part of your Docker image and start the Agent as a different process in your container. +2. Set the following configuration option in your Heroku application, to ensure that Datadog reports it correctly as a Heroku dyno: + +```shell +heroku config:add DD_HEROKU_DYNO=true +``` As an example, if you are building your Docker image using a Debian based OS, add the following lines to your `Dockerfile`: From b041c8009996b9a242e231b77d4522dc31eb6f4f Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Fri, 15 Jul 2022 10:09:45 +0200 Subject: [PATCH 136/175] Pin agent version 7.37.1 --- bin/compile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/compile b/bin/compile index ffc6a74..51e0f86 100644 --- a/bin/compile +++ b/bin/compile @@ -11,8 +11,8 @@ set -o pipefail # set -x # Set agent pinned version -DD_AGENT_PINNED_VERSION_6="6.36.0-1" -DD_AGENT_PINNED_VERSION_7="7.36.0-1" +DD_AGENT_PINNED_VERSION_6="6.37.1-1" +DD_AGENT_PINNED_VERSION_7="7.37.1-1" # Parse and derive params BUILD_DIR=$1 From 27fd2cf1dcac58d27e75353eba7f5b963b8d5420 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Fri, 15 Jul 2022 14:33:11 +0200 Subject: [PATCH 137/175] Prepare release 2.3 --- CHANGELOG.md | 5 +++++ extra/datadog.sh | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 530b7d3..1393863 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [2.3] - 2022-07-15 + +### Added +- Datadog agent pinned versions are now `6.37.1` and `7.37.1` + ## [2.2] - 2022-05-25 ### Added diff --git a/extra/datadog.sh b/extra/datadog.sh index f66e737..5b3a484 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -64,7 +64,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="dev" +BUILDPACKVERSION="2.3" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # We want always to have the Dyno ID as a host alias to improve correlation From 716444132a108bcb4778f5859b0956ce259364af Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Fri, 15 Jul 2022 15:39:23 +0200 Subject: [PATCH 138/175] Move buildpack version back to dev --- extra/datadog.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index 5b3a484..f66e737 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -64,7 +64,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="2.3" +BUILDPACKVERSION="dev" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # We want always to have the Dyno ID as a host alias to improve correlation From 6c928ff4b4bcaf87698f9215617b7877f2211c6f Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 27 Jul 2022 09:45:09 +0200 Subject: [PATCH 139/175] pin versions 6.38 and 7.38 --- bin/compile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/compile b/bin/compile index 51e0f86..0a5883b 100644 --- a/bin/compile +++ b/bin/compile @@ -11,8 +11,8 @@ set -o pipefail # set -x # Set agent pinned version -DD_AGENT_PINNED_VERSION_6="6.37.1-1" -DD_AGENT_PINNED_VERSION_7="7.37.1-1" +DD_AGENT_PINNED_VERSION_6="6.38.0-1" +DD_AGENT_PINNED_VERSION_7="7.38.0-1" # Parse and derive params BUILD_DIR=$1 From 84451711a9452f4426d89ce3ea02da0881058da7 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 27 Jul 2022 10:22:21 +0200 Subject: [PATCH 140/175] Change instructions to deploy to Heroku Change instructions to deploy to Heroku from `master` branch to `main` --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2255f3c..b4e30a9 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ heroku config:add DD_API_KEY=$DD_API_KEY # Deploy to Heroku forcing a rebuild git commit --allow-empty -m "Rebuild slug" -git push heroku master +git push heroku main ``` Once complete, the Datadog Agent is started automatically when each dyno starts. @@ -88,7 +88,7 @@ heroku config:set DD_AGENT_VERSION= -a # Rebuild your slug with the new Agent version: git commit --allow-empty -m "Rebuild slug" -git push heroku master +git push heroku main ``` ## Configuration From 9b91ef6d184cde0ef979c28da5d6485b1ea8b35a Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 27 Jul 2022 16:02:33 +0200 Subject: [PATCH 141/175] Remove unneeded system-probe binaries --- bin/compile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/compile b/bin/compile index 51e0f86..aa22b7f 100644 --- a/bin/compile +++ b/bin/compile @@ -202,6 +202,8 @@ done # Remove the system-probe binary, only needed for network performance monitoring rm -rf "$APT_DIR/opt/datadog-agent/embedded/bin/system-probe" +rm -rf "$APT_DIR/opt/datadog-agent/embedded/bin/clang-bpf" +rm -rf "$APT_DIR/opt/datadog-agent/embedded/bin/llc-bpf" rm -rf "$APT_DIR"/opt/datadog-agent/embedded/lib/libbcc* rm -rf "$APT_DIR"/opt/datadog-agent/embedded/nikos/ From e0c3f98a330dbd4dac9fe0fbcdc2839a53e36f2d Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 27 Jul 2022 19:16:29 +0200 Subject: [PATCH 142/175] Prepare release 2.4 of the buildpack --- CHANGELOG.md | 8 ++++++++ extra/datadog.sh | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1393863..7d63232 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [2.4] - 2022-07-27 + +### Added +- Datadog agent pinned versions are now `6.38.0` and `7.38.0` + +### Fixed +- Removed new introduced unneeded binaries that were increasing the footprint + ## [2.3] - 2022-07-15 ### Added diff --git a/extra/datadog.sh b/extra/datadog.sh index f66e737..a9fed88 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -64,7 +64,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="dev" +BUILDPACKVERSION="2.4" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # We want always to have the Dyno ID as a host alias to improve correlation From 7ef31c4dbc4a0c8b130177a60dc98d468e4b305b Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 27 Jul 2022 19:40:53 +0200 Subject: [PATCH 143/175] Move buildpack version back to dev --- extra/datadog.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index a9fed88..f66e737 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -64,7 +64,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="2.4" +BUILDPACKVERSION="dev" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # We want always to have the Dyno ID as a host alias to improve correlation From 5a2fecc1765e206b3577c9710abe044612eb0f74 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Thu, 25 Aug 2022 09:42:13 +0200 Subject: [PATCH 144/175] Add warning messages when removing the apm or process agent --- bin/compile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bin/compile b/bin/compile index a0aea9d..682c93d 100644 --- a/bin/compile +++ b/bin/compile @@ -217,6 +217,7 @@ rm -rf "$APT_DIR/opt/datadog-agent/embedded/bin/security-agent" if [ -f "$ENV_DIR/DD_APM_ENABLED" ]; then DD_APM_ENABLED=$(cat "$ENV_DIR/DD_APM_ENABLED") if [ "$DD_APM_ENABLED" == "false" ]; then + topic "DD_APM_ENABLE set to false. Removing the trace agent." rm -rf "$APT_DIR/opt/datadog-agent/embedded/bin/trace-agent" fi fi @@ -225,9 +226,11 @@ fi if [ -f "$ENV_DIR/DD_PROCESS_AGENT" ]; then DD_PROCESS_AGENT=$(cat "$ENV_DIR/DD_PROCESS_AGENT") if [ "$DD_PROCESS_AGENT" == "false" ]; then + topic "DD_PROCESS_AGENT set to false. Removing the process agent." rm -rf "$APT_DIR/opt/datadog-agent/embedded/bin/process-agent" fi else + topic "DD_PROCESS_AGENT not set. Removing the process agent." rm -rf "$APT_DIR/opt/datadog-agent/embedded/bin/process-agent" fi From 961e261f4aca2f44d89b11c6d969ce683f9917a5 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Thu, 25 Aug 2022 09:52:22 +0200 Subject: [PATCH 145/175] Fix typo --- bin/compile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/compile b/bin/compile index 682c93d..df64cbd 100644 --- a/bin/compile +++ b/bin/compile @@ -217,7 +217,7 @@ rm -rf "$APT_DIR/opt/datadog-agent/embedded/bin/security-agent" if [ -f "$ENV_DIR/DD_APM_ENABLED" ]; then DD_APM_ENABLED=$(cat "$ENV_DIR/DD_APM_ENABLED") if [ "$DD_APM_ENABLED" == "false" ]; then - topic "DD_APM_ENABLE set to false. Removing the trace agent." + topic "DD_APM_ENABLED set to false. Removing the trace agent." rm -rf "$APT_DIR/opt/datadog-agent/embedded/bin/trace-agent" fi fi From 0ce4d759510a5e9d2ad17a13f88a9182dc37a972 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Mon, 12 Sep 2022 17:37:03 +0200 Subject: [PATCH 146/175] Pin 6.39 and 7.39 --- bin/compile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/compile b/bin/compile index df64cbd..b850f3e 100644 --- a/bin/compile +++ b/bin/compile @@ -11,8 +11,8 @@ set -o pipefail # set -x # Set agent pinned version -DD_AGENT_PINNED_VERSION_6="6.38.0-1" -DD_AGENT_PINNED_VERSION_7="7.38.0-1" +DD_AGENT_PINNED_VERSION_6="6.39.0-1" +DD_AGENT_PINNED_VERSION_7="7.39.0-1" # Parse and derive params BUILD_DIR=$1 From 9f9629932372b82aad85fb1103607d2303926f7f Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Mon, 12 Sep 2022 18:01:20 +0200 Subject: [PATCH 147/175] Prepare release 2.5 --- CHANGELOG.md | 6 ++++++ extra/datadog.sh | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d63232..a6e16c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [2.5] - 2022-09-13 + +### Added +- Datadog agent pinned versions are now `6.39.0` and `7.39.0` +- Added warning message when removing the APM or process agent + ## [2.4] - 2022-07-27 ### Added diff --git a/extra/datadog.sh b/extra/datadog.sh index f66e737..17d6e80 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -64,7 +64,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="dev" +BUILDPACKVERSION="2.5" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # We want always to have the Dyno ID as a host alias to improve correlation From e6d74405314c6194dc90e77f3463cd1d275131c0 Mon Sep 17 00:00:00 2001 From: Caio Ariede Date: Tue, 13 Sep 2022 09:47:41 -0300 Subject: [PATCH 148/175] Fix conflict with apt buildpack Fixes an issue that happens when this buildpack is used in conjunction with https://github.com/heroku/heroku-buildpack-apt that causes the Datadog package to be installed twice, thus increasing the slug size *Steps to reproduce* 1. Create a slug with two buildpacks: https://github.com/heroku/heroku-buildpack-apt https://github.com/DataDog/heroku-buildpack-datadog 2. Trigger a build twice *Expected results* * The first build will not use cache since it does not exist yet * The second build will use the cache * In both cases, the slug size should be the same *What really happens* The second build produces a bigger slug than the first one *Why?* During the first build, the `heroku-buildpack-datadog` buildpack leaves behind a `.deb` file that it creates here: https://github.com/DataDog/heroku-buildpack-datadog/blob/17e34c7a517f9ae6047175d46a1c5e64754641ef/bin/compile#L172-L176 During the second build, by the time that the `heroku-buildpack-apt` buildpack runs, it will install all the `.deb` packages in the cache causing the Datadog Agent to be installed again: https://github.com/heroku/heroku-buildpack-apt/blob/0a778933873c3b4c4e5f046db41890ea609afb74/bin/compile#L98-L101 *Proposed fix* Just remove the `.deb` file created by Datadog after the `dpkg` installation, since it is not used directly in the next cached run --- bin/compile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/compile b/bin/compile index df64cbd..811cb26 100644 --- a/bin/compile +++ b/bin/compile @@ -183,7 +183,8 @@ dpkg -x "$DEB" "$APT_DIR" rm -rf "$APT_DIR"/opt/datadog-agent/sources \ "$APT_DIR"/opt/datadog-agent/embedded/share/doc \ "$APT_DIR"/opt/datadog-agent/embedded/share/man \ - "$APT_DIR"/opt/datadog-agent/embedded/.installed_by_pkg.txt + "$APT_DIR"/opt/datadog-agent/embedded/.installed_by_pkg.txt \ + "$DEB" # Remove Cryptodome Selftest rm -rf "$APT_DIR"/opt/datadog-agent/embedded/lib/python*/site-packages/Cryptodome/SelfTest From 210073080ff0325ac9c0d6f4ff7041aecad7af7d Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Tue, 13 Sep 2022 17:09:24 +0200 Subject: [PATCH 149/175] Revert version back to dev --- extra/datadog.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index 17d6e80..f66e737 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -64,7 +64,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="2.5" +BUILDPACKVERSION="dev" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # We want always to have the Dyno ID as a host alias to improve correlation From 36d033f1ec8032fc6df3c60a2e9d5e6d1f5b1818 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Fri, 23 Sep 2022 14:24:43 +0200 Subject: [PATCH 150/175] Match integrations configuration folder structure with default agent --- extra/datadog.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index f66e737..f908c12 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -41,10 +41,19 @@ APP_DATADOG_CONF_DIR="$APP_DATADOG/conf.d" APP_DATADOG_CHECKS_DIR="$APP_DATADOG/checks.d" # Agent integrations configuration +for dir in "$APP_DATADOG_CONF_DIR"/*; do + test -d "$dir" || continue # only match directories + dirname="$(basename -- "$dir")" + echo "Dirname ${dirname}" + cp -R "$dir" "$DD_CONF_DIR/conf.d/" +done + +# Agent integrations configuration - Deprecated for file in "$APP_DATADOG_CONF_DIR"/*.yaml; do - test -e "$file" || continue # avoid errors when glob doesn't match anything + test -f "$file" || continue # avoid errors when glob doesn't match anything filename="$(basename -- "$file")" filename="${filename%.*}" + echo "Filename ${filename}" mkdir -p "$DD_CONF_DIR/conf.d/${filename}.d" cp "$file" "$DD_CONF_DIR/conf.d/${filename}.d/conf.yaml" done From a9cc9ac5cd41cc4d438c61d75957c8bbd4e2d50b Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Tue, 27 Sep 2022 15:21:31 +0100 Subject: [PATCH 151/175] Update docs on enabling integrations --- README.md | 53 ++++++++++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index b4e30a9..793d69b 100644 --- a/README.md +++ b/README.md @@ -154,9 +154,15 @@ In order to collect system metrics for your dynos, you must: ## Enabling integrations -To enable a [Datadog- integration][17], create a file in the datadog configuration folder within your application. During the dyno start up, your YAML files are copied to the appropriate Datadog Agent configuration directories. +To enable a [Datadog- integration][17]: -For example, to enable the [Datadog-Redis integration][18], add the file `/datadog/conf.d/redisdb.yaml` at the root of your application (or `/$DD_HEROKU_CONF_FOLDER/conf.d/redisdb.yaml` if you have changed this [configuration option](#configuration)): +* Create a `datadog/conf.d` folder within your application. +* For each integration to enable, create an `.d` folder +* Under that folder, create a `conf.yaml` with the [configuration for the integration][18]. + +During the dyno start up, your YAML files are copied to the appropriate Datadog Agent configuration directories. + +For example, to enable the [Datadog-Redis integration][19], add the file `/datadog/conf.d/redisdb.d/conf.yaml` at the root of your application (or `/$DD_HEROKU_CONF_FOLDER/conf.d/redisdb.d/conf.yaml` if you have changed this [configuration option](#configuration)): ```yaml init_config: @@ -174,17 +180,17 @@ instances: port: 6379 ``` -**Note**: See the sample [redisdb.d/conf.yaml][19] for all available configuration options. +**Note**: See the sample [redisdb.d/conf.yaml][20] for all available configuration options. ### Community Integrations -If the integration you are enabling is part of the [Community Integrations][20], install the package as part of the [prerun script](#prerun-script). +If the integration you are enabling is part of the [Community Integrations][21], install the package as part of the [prerun script](#prerun-script). ``` agent-wrapper integration install -t datadog-== ``` -For example, to install the [ping integration][21], create the configuration file `datadog/conf.d/ping.yaml` and add the following line to your prerun script: +For example, to install the [ping integration][22], create the configuration file `datadog/conf.d/ping.d/conf.yaml` and add the following line to your prerun script: ``` agent-wrapper integration install -t datadog-ping==1.0.0 @@ -204,7 +210,7 @@ fi ## Enabling custom checks -To enable your own [Agent Custom Checks][22], create a `checks.d` folder in the datadog configuration folder within your application. Under it, copy all `.py` and `.yaml` files from your custom checks. During the dyno start up, your files are copied to the appropriate Datadog Agent configuration directories. +To enable your own [Agent Custom Checks][23], create a `checks.d` folder in the datadog configuration folder within your application. Under it, copy all `.py` and `.yaml` files from your custom checks. During the dyno start up, your files are copied to the appropriate Datadog Agent configuration directories. For example, if you have two custom checks, `foo` and `bar`, this would be the right folder tree: @@ -278,7 +284,7 @@ To reduce your slug size, make sure that `DD_APM_ENABLED` is set to `false`, if ## Debugging -To run any of the [information or debugging commands][23], use the `agent-wrapper` command. +To run any of the [information or debugging commands][24], use the `agent-wrapper` command. For example, to display the status of your Datadog Agent and enabled integrations, run: @@ -298,7 +304,7 @@ The Datadog buildpack does not collect logs from the Heroku platform. To set up ## Using Heroku with Docker images -This buildpack only works for Heroku deployments that use [Heroku's Slug Compiler][24]. If you are deploying your application in Heroku using Docker containers: +This buildpack only works for Heroku deployments that use [Heroku's Slug Compiler][25]. If you are deploying your application in Heroku using Docker containers: 1. Add the Datadog Agent as part of your Docker image and start the Agent as a different process in your container. 2. Set the following configuration option in your Heroku application, to ensure that Datadog reports it correctly as a Heroku dyno: @@ -352,15 +358,15 @@ datadog-agent run & /opt/datadog-agent/embedded/bin/process-agent --config=/etc/datadog-agent/datadog.yaml ``` -For more advanced options in the Docker image, reference the [Datadog Agent Docker files][25]. +For more advanced options in the Docker image, reference the [Datadog Agent Docker files][26]. ## Contributing -See the [contributing guidelines][26] to learn how to open an issue or PR to the [Heroku-buildpack-datadog repository][27]. +See the [contributing guidelines][27] to learn how to open an issue or PR to the [Heroku-buildpack-datadog repository][28]. ## History -Earlier versions of this project were forked from the [miketheman heroku-buildpack-datadog project][28]. It was largely rewritten for Datadog's Agent version 6. Changes and more information can be found in the [changelog][29]. +Earlier versions of this project were forked from the [miketheman heroku-buildpack-datadog project][29]. It was largely rewritten for Datadog's Agent version 6. Changes and more information can be found in the [changelog][30]. ## Troubleshooting @@ -498,15 +504,16 @@ After an upgrade of the buildpack or Agent, you must recompile your application' [15]: https://docs.datadoghq.com/logs/guide/collect-heroku-logs [16]: https://docs.datadoghq.com/logs/logs_to_metrics/ [17]: https://docs.datadoghq.com/integrations/ -[18]: https://docs.datadoghq.com/integrations/redisdb/ -[19]: https://github.com/DataDog/integrations-core/blob/master/redisdb/datadog_checks/redisdb/data/conf.yaml.example -[20]: https://github.com/DataDog/integrations-extras/ -[21]: https://github.com/DataDog/integrations-extras/tree/master/ping -[22]: https://docs.datadoghq.com/developers/custom_checks/ -[23]: https://docs.datadoghq.com/agent/guide/agent-commands/#agent-status-and-information -[24]: https://devcenter.heroku.com/articles/slug-compiler -[25]: https://github.com/DataDog/datadog-agent/tree/master/Dockerfiles -[26]: https://github.com/DataDog/heroku-buildpack-datadog/blob/master/CONTRIBUTING.md -[27]: https://github.com/DataDog/heroku-buildpack-datadog -[28]: https://github.com/miketheman/heroku-buildpack-datadog -[29]: https://github.com/DataDog/heroku-buildpack-datadog/blob/master/CHANGELOG.md +[18]: https://docs.datadoghq.com/getting_started/integrations/#configuring-agent-integrations +[19]: https://docs.datadoghq.com/integrations/redisdb/ +[20]: https://github.com/DataDog/integrations-core/blob/master/redisdb/datadog_checks/redisdb/data/conf.yaml.example +[21]: https://github.com/DataDog/integrations-extras/ +[22]: https://github.com/DataDog/integrations-extras/tree/master/ping +[23]: https://docs.datadoghq.com/developers/custom_checks/ +[24]: https://docs.datadoghq.com/agent/guide/agent-commands/#agent-status-and-information +[25]: https://devcenter.heroku.com/articles/slug-compiler +[26]: https://github.com/DataDog/datadog-agent/tree/master/Dockerfiles +[27]: https://github.com/DataDog/heroku-buildpack-datadog/blob/master/CONTRIBUTING.md +[28]: https://github.com/DataDog/heroku-buildpack-datadog +[29]: https://github.com/miketheman/heroku-buildpack-datadog +[30]: https://github.com/DataDog/heroku-buildpack-datadog/blob/master/CHANGELOG.md From 11b9f00aa91ca86f43b179860292d75dccf1b5f8 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Tue, 27 Sep 2022 15:28:01 +0100 Subject: [PATCH 152/175] Remove chatty output --- extra/datadog.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index f908c12..d22cff1 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -43,8 +43,6 @@ APP_DATADOG_CHECKS_DIR="$APP_DATADOG/checks.d" # Agent integrations configuration for dir in "$APP_DATADOG_CONF_DIR"/*; do test -d "$dir" || continue # only match directories - dirname="$(basename -- "$dir")" - echo "Dirname ${dirname}" cp -R "$dir" "$DD_CONF_DIR/conf.d/" done @@ -53,7 +51,6 @@ for file in "$APP_DATADOG_CONF_DIR"/*.yaml; do test -f "$file" || continue # avoid errors when glob doesn't match anything filename="$(basename -- "$file")" filename="${filename%.*}" - echo "Filename ${filename}" mkdir -p "$DD_CONF_DIR/conf.d/${filename}.d" cp "$file" "$DD_CONF_DIR/conf.d/${filename}.d/conf.yaml" done From 7f3e8aa250926bbe27bb9ea26fc33b4b28d9ab65 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Mon, 7 Nov 2022 14:53:54 +0100 Subject: [PATCH 153/175] Pin agent version 6/7.40 --- bin/compile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/compile b/bin/compile index b850f3e..272292b 100644 --- a/bin/compile +++ b/bin/compile @@ -11,8 +11,8 @@ set -o pipefail # set -x # Set agent pinned version -DD_AGENT_PINNED_VERSION_6="6.39.0-1" -DD_AGENT_PINNED_VERSION_7="7.39.0-1" +DD_AGENT_PINNED_VERSION_6="6.40.0-1" +DD_AGENT_PINNED_VERSION_7="7.40.0-1" # Parse and derive params BUILD_DIR=$1 From 09321797d3bac2693bc274c307ecd81a3d310403 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 9 Nov 2022 11:43:39 +0100 Subject: [PATCH 154/175] Prepare buildpack release --- CHANGELOG.md | 9 +++++++++ extra/datadog.sh | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a6e16c0..b65a566 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,15 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [2.6] - 2022-11-09 + +### Added +- Datadog agent pinned versions are now `6.40.0` and `7.40.0` +- Config files for integrations now can follow the same folder structure used in the Agent + +### Fixed +- Slug size now doesn't increae when apt buildpack is used (Thanks caioariede for the contribution!) + ## [2.5] - 2022-09-13 ### Added diff --git a/extra/datadog.sh b/extra/datadog.sh index d22cff1..09c2260 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -70,7 +70,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="dev" +BUILDPACKVERSION="2.6" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # We want always to have the Dyno ID as a host alias to improve correlation From 74611cc32960b61bd96e9f8c80ef2394d18f1a02 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 9 Nov 2022 16:36:35 +0100 Subject: [PATCH 155/175] Revert version back to dev --- extra/datadog.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index 09c2260..d22cff1 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -70,7 +70,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="2.6" +BUILDPACKVERSION="dev" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # We want always to have the Dyno ID as a host alias to improve correlation From 9866691aa55921bb0df58cb9d826aafbc19820c6 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Thu, 22 Dec 2022 11:10:51 +0100 Subject: [PATCH 156/175] Pin versions 7.41.1 and 6.41.1 --- bin/compile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/compile b/bin/compile index e050035..36fec59 100644 --- a/bin/compile +++ b/bin/compile @@ -11,8 +11,8 @@ set -o pipefail # set -x # Set agent pinned version -DD_AGENT_PINNED_VERSION_6="6.40.0-1" -DD_AGENT_PINNED_VERSION_7="7.40.0-1" +DD_AGENT_PINNED_VERSION_6="6.41.1-1" +DD_AGENT_PINNED_VERSION_7="7.41.1-1" # Parse and derive params BUILD_DIR=$1 From 1dfe8690d57b796b330e5ec16c806a326d715058 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Thu, 22 Dec 2022 11:11:06 +0100 Subject: [PATCH 157/175] Prepare release 2.7 --- CHANGELOG.md | 5 +++++ extra/datadog.sh | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b65a566..e4582e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [2.7] - 2022-12-22 + +### Added +- Datadog agent pinned versions are now `6.41.1` and `7.41.1` + ## [2.6] - 2022-11-09 ### Added diff --git a/extra/datadog.sh b/extra/datadog.sh index d22cff1..dfd3d1a 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -70,7 +70,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="dev" +BUILDPACKVERSION="2.7" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # We want always to have the Dyno ID as a host alias to improve correlation From 9d1ec66ef10a23005e71313bedf8935212d7fdb4 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Thu, 22 Dec 2022 15:27:39 +0100 Subject: [PATCH 158/175] Revert version back to dev --- extra/datadog.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index dfd3d1a..d22cff1 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -70,7 +70,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="2.7" +BUILDPACKVERSION="dev" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # We want always to have the Dyno ID as a host alias to improve correlation From 78ad6f9fc111591e6c4974406cdb26e10bdcb5f3 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 25 Jan 2023 10:20:39 +0100 Subject: [PATCH 159/175] Pin versions 7.42.0 and 6.42.0 --- bin/compile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/compile b/bin/compile index 36fec59..870dbfb 100644 --- a/bin/compile +++ b/bin/compile @@ -11,8 +11,8 @@ set -o pipefail # set -x # Set agent pinned version -DD_AGENT_PINNED_VERSION_6="6.41.1-1" -DD_AGENT_PINNED_VERSION_7="7.41.1-1" +DD_AGENT_PINNED_VERSION_6="6.42.0-1" +DD_AGENT_PINNED_VERSION_7="7.42.0-1" # Parse and derive params BUILD_DIR=$1 From 6dd832e69a5a6ae157bb7df42825c57400465c72 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 25 Jan 2023 16:21:59 +0100 Subject: [PATCH 160/175] Prepare 2.8 release --- CHANGELOG.md | 9 +++++++++ extra/datadog.sh | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e4582e2..608b16d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,15 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [2.8] - 2022-01-25 + +### Removed +- Installing checks with `agent-wrapper integration install` is no longer supported for Datadog Agent 6.x.\ +Upgrade to Datadog Agent 7.x by setting the environment variable `DD_AGENT_MAJOR_VERSION=7` in your Heroku application and rebuilding the slug. + +### Added +- Datadog agent pinned versions are now `6.42.0` and `7.42.0` + ## [2.7] - 2022-12-22 ### Added diff --git a/extra/datadog.sh b/extra/datadog.sh index d22cff1..4669b9c 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -70,7 +70,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="dev" +BUILDPACKVERSION="2.8" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # We want always to have the Dyno ID as a host alias to improve correlation From 9c045cfc9e1f9e39cb4d18df60b3c3a9bfe88f49 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 25 Jan 2023 16:53:56 +0100 Subject: [PATCH 161/175] Revert buildpack version back to dev --- CHANGELOG.md | 2 +- extra/datadog.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 608b16d..8803d28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). -## [2.8] - 2022-01-25 +## [2.8] - 2023-01-25 ### Removed - Installing checks with `agent-wrapper integration install` is no longer supported for Datadog Agent 6.x.\ diff --git a/extra/datadog.sh b/extra/datadog.sh index 4669b9c..d22cff1 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -70,7 +70,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="2.8" +BUILDPACKVERSION="dev" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # We want always to have the Dyno ID as a host alias to improve correlation From a54ec8c08c2c9f1a9e005c8c8f940a245b89f437 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Tue, 7 Feb 2023 11:52:10 +0100 Subject: [PATCH 162/175] Automatically enable Heroku postgres integration --- extra/datadog.sh | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/extra/datadog.sh b/extra/datadog.sh index d22cff1..430a542 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -10,6 +10,9 @@ DD_LOG_DIR="$APT_DIR/var/log/datadog" DD_CONF_DIR="$APT_DIR/etc/datadog-agent" DD_INSTALL_INFO="$DD_CONF_DIR/install_info" export DATADOG_CONF="$DD_CONF_DIR/datadog.yaml" +export INTEGRATIONS_CONF="$DD_CONF_DIR/conf.d" +export POSTGRES_CONF="$INTEGRATIONS_CONF/postgres.d" +export REDIS_CONF="$INTEGRATIONS_CONF/redisdb.d" # Update Env Vars with new paths for apt packages export PATH="$APT_DIR/usr/bin:$DD_BIN_DIR:$PATH" @@ -170,6 +173,26 @@ DD_PYTHONPATH="$DD_PYTHONPATH:$SETUPTOOLS_PATH:$PIP_PATH" # Export agent's PYTHONPATH be used by the agent-wrapper export DD_PYTHONPATH="$DD_DIR/embedded/lib:$DD_PYTHONPATH" +## Default integrations configuration + +# Update the Postgres configuration from above using the Heroku application environment variable +if [ "$ENABLE_HEROKU_POSTGRES" == "true" ]; then + + cp "$POSTGRES_CONF/conf.yaml.example" "$POSTGRES_CONF/conf.yaml" + + if [ -n "$DATABASE_URL" ]; then + POSTGREGEX='^postgres://([^:]+):([^@]+)@([^:]+):([^/]+)/(.*)$' + if [[ $DATABASE_URL =~ $POSTGREGEX ]]; then + sed -i "s/^ - host:.*/ - host: ${BASH_REMATCH[3]}/" "$POSTGRES_CONF/conf.yaml" + sed -i "s/^ username:.*/ username: ${BASH_REMATCH[1]}/" "$POSTGRES_CONF/conf.yaml" + sed -i "s/^ # password:.*/ password: ${BASH_REMATCH[2]}/" "$POSTGRES_CONF/conf.yaml" + sed -i "s/^ # port:.*/ port: ${BASH_REMATCH[4]}/" "$POSTGRES_CONF/conf.yaml" + sed -i "s/^ # dbname:.*/ dbname: ${BASH_REMATCH[5]}/" "$POSTGRES_CONF/conf.yaml" + sed -i "s/^ # ssl:.*/ ssl: True/" "$POSTGRES_CONF/conf.yaml" + fi + fi +fi + # Give applications a chance to modify env vars prior to running. # Note that this can modify existing env vars or perform other actions (e.g. modify the conf file). # For more information on variables and other things you may wish to modify, reference this script From 617a731db83199c39edf0f51fa8c48d5d68d30e8 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Tue, 7 Feb 2023 12:42:47 +0100 Subject: [PATCH 163/175] Automatically enable Heroku redis integration --- extra/datadog.sh | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/extra/datadog.sh b/extra/datadog.sh index 430a542..1817590 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -193,6 +193,26 @@ if [ "$ENABLE_HEROKU_POSTGRES" == "true" ]; then fi fi +# Update the Redis configuration from above using the Heroku application environment variable +if [ "$ENABLE_HEROKU_REDIS" == "true" ]; then + + cp "$REDIS_CONF/conf.yaml.example" "$REDIS_CONF/conf.yaml" + + if [ -n "$REDIS_TLS_URL" ]; then + REDISREGEX='^rediss://([^:]*):([^@]+)@([^:]+):(.+)$' + if [[ $REDIS_TLS_URL =~ $REDISREGEX ]]; then + sed -i "s/^ - host:.*/ - host: ${BASH_REMATCH[3]}/" "$REDIS_CONF/conf.yaml" + sed -i "s/^ # password:.*/ password: ${BASH_REMATCH[2]}/" "$REDIS_CONF/conf.yaml" + sed -i "s/^ port:.*/ port: ${BASH_REMATCH[4]}/" "$REDIS_CONF/conf.yaml" + sed -i "s/^ # ssl:.*/ ssl: True/" "$REDIS_CONF/conf.yaml" + sed -i "s/^ # ssl_cert_reqs:.*/ ssl_cert_reqs: 0/" "$REDIS_CONF/conf.yaml" + if [[ ! -z ${BASH_REMATCH[1]} ]]; then + sed -i "s/^# username:.*/ username: ${BASH_REMATCH[1]}/" "$REDIS_CONF/conf.yaml" + fi + fi + fi +fi + # Give applications a chance to modify env vars prior to running. # Note that this can modify existing env vars or perform other actions (e.g. modify the conf file). # For more information on variables and other things you may wish to modify, reference this script From 2691d01a73825225f5a5ae07116614a71d04d0d1 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Tue, 7 Feb 2023 13:06:28 +0100 Subject: [PATCH 164/175] Support both TLS and non TLS redis connections --- extra/datadog.sh | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index 1817590..7ba8a8c 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -198,16 +198,18 @@ if [ "$ENABLE_HEROKU_REDIS" == "true" ]; then cp "$REDIS_CONF/conf.yaml.example" "$REDIS_CONF/conf.yaml" - if [ -n "$REDIS_TLS_URL" ]; then - REDISREGEX='^rediss://([^:]*):([^@]+)@([^:]+):(.+)$' - if [[ $REDIS_TLS_URL =~ $REDISREGEX ]]; then - sed -i "s/^ - host:.*/ - host: ${BASH_REMATCH[3]}/" "$REDIS_CONF/conf.yaml" - sed -i "s/^ # password:.*/ password: ${BASH_REMATCH[2]}/" "$REDIS_CONF/conf.yaml" - sed -i "s/^ port:.*/ port: ${BASH_REMATCH[4]}/" "$REDIS_CONF/conf.yaml" - sed -i "s/^ # ssl:.*/ ssl: True/" "$REDIS_CONF/conf.yaml" - sed -i "s/^ # ssl_cert_reqs:.*/ ssl_cert_reqs: 0/" "$REDIS_CONF/conf.yaml" + if [ -n "$REDIS_URL" ]; then + REDISREGEX='^redis(s?)://([^:]*):([^@]+)@([^:]+):(.+)$' + if [[ $REDIS_URL =~ $REDISREGEX ]]; then + sed -i "s/^ - host:.*/ - host: ${BASH_REMATCH[4]}/" "$REDIS_CONF/conf.yaml" + sed -i "s/^ # password:.*/ password: ${BASH_REMATCH[3]}/" "$REDIS_CONF/conf.yaml" + sed -i "s/^ port:.*/ port: ${BASH_REMATCH[5]}/" "$REDIS_CONF/conf.yaml" if [[ ! -z ${BASH_REMATCH[1]} ]]; then - sed -i "s/^# username:.*/ username: ${BASH_REMATCH[1]}/" "$REDIS_CONF/conf.yaml" + sed -i "s/^ # ssl:.*/ ssl: True/" "$REDIS_CONF/conf.yaml" + sed -i "s/^ # ssl_cert_reqs:.*/ ssl_cert_reqs: 0/" "$REDIS_CONF/conf.yaml" + fi + if [[ ! -z ${BASH_REMATCH[2]} ]]; then + sed -i "s/^ # username:.*/ username: ${BASH_REMATCH[2]}/" "$REDIS_CONF/conf.yaml" fi fi fi From 28f1658f3b1015ad28ec1734c8a9c0cd85f4a1bb Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 8 Feb 2023 09:47:32 +0100 Subject: [PATCH 165/175] Allow changing the env variable with the Redis URL --- extra/datadog.sh | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index 7ba8a8c..9b8bf14 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -195,12 +195,17 @@ fi # Update the Redis configuration from above using the Heroku application environment variable if [ "$ENABLE_HEROKU_REDIS" == "true" ]; then + # The connection URL is, by default REDIS_URL, but can be configured by the user + REDIS_URL="REDIS_URL" + if [[ ! -z ${REDIS_URL_VAR} ]]; then + REDIS_URL=${REDIS_URL_VAR} + fi cp "$REDIS_CONF/conf.yaml.example" "$REDIS_CONF/conf.yaml" - if [ -n "$REDIS_URL" ]; then - REDISREGEX='^redis(s?)://([^:]*):([^@]+)@([^:]+):(.+)$' - if [[ $REDIS_URL =~ $REDISREGEX ]]; then + if [ -n "${!REDIS_URL}" ]; then + REDISREGEX='^redis(s?)://([^:]*):([^@]+)@([^:]+):([^/]+)/?(.*)$' + if [[ ${!REDIS_URL} =~ $REDISREGEX ]]; then sed -i "s/^ - host:.*/ - host: ${BASH_REMATCH[4]}/" "$REDIS_CONF/conf.yaml" sed -i "s/^ # password:.*/ password: ${BASH_REMATCH[3]}/" "$REDIS_CONF/conf.yaml" sed -i "s/^ port:.*/ port: ${BASH_REMATCH[5]}/" "$REDIS_CONF/conf.yaml" @@ -211,6 +216,9 @@ if [ "$ENABLE_HEROKU_REDIS" == "true" ]; then if [[ ! -z ${BASH_REMATCH[2]} ]]; then sed -i "s/^ # username:.*/ username: ${BASH_REMATCH[2]}/" "$REDIS_CONF/conf.yaml" fi + if [[ ! -z ${BASH_REMATCH[6]} ]]; then + sed -i "s/^ # db:.*/ db: ${BASH_REMATCH[6]}/" "$REDIS_CONF/conf.yaml" + fi fi fi fi From b0adbc12b344e4e461a9799de9eb2cf3e43f808b Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 8 Feb 2023 10:25:26 +0100 Subject: [PATCH 166/175] Allow changing the env variable with the Postgres URL --- extra/datadog.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index 9b8bf14..da56048 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -177,12 +177,17 @@ export DD_PYTHONPATH="$DD_DIR/embedded/lib:$DD_PYTHONPATH" # Update the Postgres configuration from above using the Heroku application environment variable if [ "$ENABLE_HEROKU_POSTGRES" == "true" ]; then + # The connection URL is, by default DATABASE_URL, but can be configured by the user + DATABASE_URL="DATABASE_URL" + if [[ ! -z ${POSTGRES_URL_VAR} ]]; then + DATABASE_URL=${POSTGRES_URL_VAR} + fi cp "$POSTGRES_CONF/conf.yaml.example" "$POSTGRES_CONF/conf.yaml" - if [ -n "$DATABASE_URL" ]; then + if [ -n "${!DATABASE_URL}" ]; then POSTGREGEX='^postgres://([^:]+):([^@]+)@([^:]+):([^/]+)/(.*)$' - if [[ $DATABASE_URL =~ $POSTGREGEX ]]; then + if [[ ${!DATABASE_URL} =~ $POSTGREGEX ]]; then sed -i "s/^ - host:.*/ - host: ${BASH_REMATCH[3]}/" "$POSTGRES_CONF/conf.yaml" sed -i "s/^ username:.*/ username: ${BASH_REMATCH[1]}/" "$POSTGRES_CONF/conf.yaml" sed -i "s/^ # password:.*/ password: ${BASH_REMATCH[2]}/" "$POSTGRES_CONF/conf.yaml" From 1a56147fff35a0d45ab6a31067d3b18475f07016 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 8 Feb 2023 11:11:25 +0100 Subject: [PATCH 167/175] Update documentation around enabling integrations --- README.md | 66 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 39 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 793d69b..baf07b8 100644 --- a/README.md +++ b/README.md @@ -154,7 +154,37 @@ In order to collect system metrics for your dynos, you must: ## Enabling integrations -To enable a [Datadog- integration][17]: +### Enabling the Datadog Redis integration + +If you are using a Redis addon in your Heroku application (for example, Heroku Data for Redis or Redis Enterprise Cloud), you can easily enable the Datadog Redis integration by setting an envionment variable: + +``` +heroku config:set ENABLE_HEROKU_REDIS=true +``` + +By default, this integration assumes the Redis connection URL is defined in an environement variable called `REDIS_URL` (this is default for Heroku Data for Redis and other addons). + +If your connection URL is defined in a different environment variable, set the `REDIS_URL_VAR` environment variable to the name of the variable name. For example, if you're using Redis Enterprise Cloud, set it to "REDISCLOUD_URL": + +``` +heroku config:set REDIS_URL_VAR=REDISCLOUD_URL +``` + +### Enabling the Datadog Postgres integration + +If you are using a Postgres addon in your Heroku application (for example, Heroku Postgres), you can easily enable the Datadog Postgres integration by setting an envionment variable: + +``` +heroku config:set ENABLE_HEROKU_POSTGRES=true +``` + +By default, this integration assumes the Postgres connection URL is defined in an environement variable called `DATABASE_URL` (this is default for Heroku Postgres and other addons). + +If your connection URL is defined in a different environment variable, set the `POSTGRES_URL_VAR` environment variable to the name of the variable name. + +### Enabling other integrations + +To enable any [Datadog- integration][17]: * Create a `datadog/conf.d` folder within your application. * For each integration to enable, create an `.d` folder @@ -162,25 +192,19 @@ To enable a [Datadog- integration][17]: During the dyno start up, your YAML files are copied to the appropriate Datadog Agent configuration directories. -For example, to enable the [Datadog-Redis integration][19], add the file `/datadog/conf.d/redisdb.d/conf.yaml` at the root of your application (or `/$DD_HEROKU_CONF_FOLDER/conf.d/redisdb.d/conf.yaml` if you have changed this [configuration option](#configuration)): +For example, to enable the [Datadog-Memcache integration][19], add the file `/datadog/conf.d/mcache.d/conf.yaml` at the root of your application (or `/$DD_HEROKU_CONF_FOLDER/conf.d/mcache.d/conf.yaml` if you have changed this [configuration option](#configuration)): ```yaml init_config: instances: - - ## @param host - string - required - ## Enter the host to connect to. - # - - host: - - ## @param port - integer - required - ## Enter the port of the host to connect to. - # - port: 6379 + ## @param url - string - required + ## url used to connect to the Memcached instance. + # + - url: localhost ``` -**Note**: See the sample [redisdb.d/conf.yaml][20] for all available configuration options. +**Note**: See the sample [mcache.d/conf.yaml][20] for all available configuration options. ### Community Integrations @@ -250,18 +274,6 @@ if [ -n "$HEROKU_SLUG_COMMIT" ]; then DD_VERSION=$HEROKU_SLUG_COMMIT fi -# Update the Postgres configuration from above using the Heroku application environment variable -if [ -n "$DATABASE_URL" ]; then - POSTGREGEX='^postgres://([^:]+):([^@]+)@([^:]+):([^/]+)/(.*)$' - if [[ $DATABASE_URL =~ $POSTGREGEX ]]; then - sed -i "s//${BASH_REMATCH[3]}/" "$DD_CONF_DIR/conf.d/postgres.d/conf.yaml" - sed -i "s//${BASH_REMATCH[1]}/" "$DD_CONF_DIR/conf.d/postgres.d/conf.yaml" - sed -i "s//${BASH_REMATCH[2]}/" "$DD_CONF_DIR/conf.d/postgres.d/conf.yaml" - sed -i "s//${BASH_REMATCH[4]}/" "$DD_CONF_DIR/conf.d/postgres.d/conf.yaml" - sed -i "s//${BASH_REMATCH[5]}/" "$DD_CONF_DIR/conf.d/postgres.d/conf.yaml" - fi -fi - # Install the "ping" community integration agent-wrapper integration install -t datadog-ping==1.0.0 ``` @@ -505,8 +517,8 @@ After an upgrade of the buildpack or Agent, you must recompile your application' [16]: https://docs.datadoghq.com/logs/logs_to_metrics/ [17]: https://docs.datadoghq.com/integrations/ [18]: https://docs.datadoghq.com/getting_started/integrations/#configuring-agent-integrations -[19]: https://docs.datadoghq.com/integrations/redisdb/ -[20]: https://github.com/DataDog/integrations-core/blob/master/redisdb/datadog_checks/redisdb/data/conf.yaml.example +[19]: https://docs.datadoghq.com/integrations/mcache/ +[20]: https://github.com/DataDog/integrations-core/blob/master/mcache/datadog_checks/mcache/data/conf.yaml.example [21]: https://github.com/DataDog/integrations-extras/ [22]: https://github.com/DataDog/integrations-extras/tree/master/ping [23]: https://docs.datadoghq.com/developers/custom_checks/ From 1b869c1ea5f9cbe7efdfbd368d461603ada70792 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 8 Feb 2023 12:09:04 +0100 Subject: [PATCH 168/175] Fix default env var for connection url logic --- extra/datadog.sh | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index da56048..0bb0575 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -178,16 +178,15 @@ export DD_PYTHONPATH="$DD_DIR/embedded/lib:$DD_PYTHONPATH" # Update the Postgres configuration from above using the Heroku application environment variable if [ "$ENABLE_HEROKU_POSTGRES" == "true" ]; then # The connection URL is, by default DATABASE_URL, but can be configured by the user - DATABASE_URL="DATABASE_URL" - if [[ ! -z ${POSTGRES_URL_VAR} ]]; then - DATABASE_URL=${POSTGRES_URL_VAR} + if [[ -z ${POSTGRES_URL_VAR} ]]; then + POSTGRES_URL_VAR="DATABASE_URL" fi cp "$POSTGRES_CONF/conf.yaml.example" "$POSTGRES_CONF/conf.yaml" - if [ -n "${!DATABASE_URL}" ]; then + if [ -n "${!POSTGRES_URL_VAR}" ]; then POSTGREGEX='^postgres://([^:]+):([^@]+)@([^:]+):([^/]+)/(.*)$' - if [[ ${!DATABASE_URL} =~ $POSTGREGEX ]]; then + if [[ ${!POSTGRES_URL_VAR} =~ $POSTGREGEX ]]; then sed -i "s/^ - host:.*/ - host: ${BASH_REMATCH[3]}/" "$POSTGRES_CONF/conf.yaml" sed -i "s/^ username:.*/ username: ${BASH_REMATCH[1]}/" "$POSTGRES_CONF/conf.yaml" sed -i "s/^ # password:.*/ password: ${BASH_REMATCH[2]}/" "$POSTGRES_CONF/conf.yaml" @@ -201,16 +200,15 @@ fi # Update the Redis configuration from above using the Heroku application environment variable if [ "$ENABLE_HEROKU_REDIS" == "true" ]; then # The connection URL is, by default REDIS_URL, but can be configured by the user - REDIS_URL="REDIS_URL" - if [[ ! -z ${REDIS_URL_VAR} ]]; then - REDIS_URL=${REDIS_URL_VAR} + if [[ -z ${REDIS_URL_VAR} ]]; then + REDIS_URL_VAR="REDIS_URL" fi cp "$REDIS_CONF/conf.yaml.example" "$REDIS_CONF/conf.yaml" - if [ -n "${!REDIS_URL}" ]; then + if [ -n "${!REDIS_URL_VAR}" ]; then REDISREGEX='^redis(s?)://([^:]*):([^@]+)@([^:]+):([^/]+)/?(.*)$' - if [[ ${!REDIS_URL} =~ $REDISREGEX ]]; then + if [[ ${!REDIS_URL_VAR} =~ $REDISREGEX ]]; then sed -i "s/^ - host:.*/ - host: ${BASH_REMATCH[4]}/" "$REDIS_CONF/conf.yaml" sed -i "s/^ # password:.*/ password: ${BASH_REMATCH[3]}/" "$REDIS_CONF/conf.yaml" sed -i "s/^ port:.*/ port: ${BASH_REMATCH[5]}/" "$REDIS_CONF/conf.yaml" From 8c360a2fad2c3ee408f634f8aca19ec8f443e5d1 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Wed, 8 Feb 2023 12:36:14 +0100 Subject: [PATCH 169/175] Send heroku app tags with postgres metrics --- extra/datadog.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/extra/datadog.sh b/extra/datadog.sh index 0bb0575..9f7c3f9 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -193,6 +193,7 @@ if [ "$ENABLE_HEROKU_POSTGRES" == "true" ]; then sed -i "s/^ # port:.*/ port: ${BASH_REMATCH[4]}/" "$POSTGRES_CONF/conf.yaml" sed -i "s/^ # dbname:.*/ dbname: ${BASH_REMATCH[5]}/" "$POSTGRES_CONF/conf.yaml" sed -i "s/^ # ssl:.*/ ssl: True/" "$POSTGRES_CONF/conf.yaml" + sed -i "s/^ disable_generic_tags:.*/ disable_generic_tags: false/" "$POSTGRES_CONF/conf.yaml" fi fi fi From 02ec317d846d1773ba56edbed826e5520ecd7ebf Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Thu, 9 Feb 2023 09:46:23 +0100 Subject: [PATCH 170/175] Apply suggestions from the documentation team Co-authored-by: Bryce Eadie --- README.md | 12 ++++++------ extra/datadog.sh | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index baf07b8..879fc6f 100644 --- a/README.md +++ b/README.md @@ -156,15 +156,15 @@ In order to collect system metrics for your dynos, you must: ### Enabling the Datadog Redis integration -If you are using a Redis addon in your Heroku application (for example, Heroku Data for Redis or Redis Enterprise Cloud), you can easily enable the Datadog Redis integration by setting an envionment variable: +If you are using a Redis add-on in your Heroku application (for example, Heroku Data for Redis or Redis Enterprise Cloud), you can enable the Datadog Redis integration by setting an environment variable: ``` heroku config:set ENABLE_HEROKU_REDIS=true ``` -By default, this integration assumes the Redis connection URL is defined in an environement variable called `REDIS_URL` (this is default for Heroku Data for Redis and other addons). +By default, this integration assumes the Redis connection URL is defined in an environment variable called `REDIS_URL` (this is the default configuration for Heroku Data for Redis and other Redis add-ons). -If your connection URL is defined in a different environment variable, set the `REDIS_URL_VAR` environment variable to the name of the variable name. For example, if you're using Redis Enterprise Cloud, set it to "REDISCLOUD_URL": +If your connection URL is defined in a different environment variable, set the `REDIS_URL_VAR` environment variable to the variable name. For example, if you're using Redis Enterprise Cloud, set it to `REDISCLOUD_URL`: ``` heroku config:set REDIS_URL_VAR=REDISCLOUD_URL @@ -172,15 +172,15 @@ heroku config:set REDIS_URL_VAR=REDISCLOUD_URL ### Enabling the Datadog Postgres integration -If you are using a Postgres addon in your Heroku application (for example, Heroku Postgres), you can easily enable the Datadog Postgres integration by setting an envionment variable: +If you are using a Postgres add-on in your Heroku application (for example, Heroku Postgres), you can enable the Datadog Postgres integration by setting an environment variable: ``` heroku config:set ENABLE_HEROKU_POSTGRES=true ``` -By default, this integration assumes the Postgres connection URL is defined in an environement variable called `DATABASE_URL` (this is default for Heroku Postgres and other addons). +By default, this integration assumes the Postgres connection URL is defined in an environment variable called `DATABASE_URL` (this is the default configuration for Heroku Postgres and other Postgres add-ons). -If your connection URL is defined in a different environment variable, set the `POSTGRES_URL_VAR` environment variable to the name of the variable name. +If your connection URL is defined in a different environment variable, set the `POSTGRES_URL_VAR` environment variable to the variable name. ### Enabling other integrations diff --git a/extra/datadog.sh b/extra/datadog.sh index 9f7c3f9..9d609c3 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -177,7 +177,7 @@ export DD_PYTHONPATH="$DD_DIR/embedded/lib:$DD_PYTHONPATH" # Update the Postgres configuration from above using the Heroku application environment variable if [ "$ENABLE_HEROKU_POSTGRES" == "true" ]; then - # The connection URL is, by default DATABASE_URL, but can be configured by the user + # The default connection URL is DATABASE_URL, but can be configured by the user if [[ -z ${POSTGRES_URL_VAR} ]]; then POSTGRES_URL_VAR="DATABASE_URL" fi @@ -200,7 +200,7 @@ fi # Update the Redis configuration from above using the Heroku application environment variable if [ "$ENABLE_HEROKU_REDIS" == "true" ]; then - # The connection URL is, by default REDIS_URL, but can be configured by the user + # The default connection URL is REDIS_URL, but can be configured by the user if [[ -z ${REDIS_URL_VAR} ]]; then REDIS_URL_VAR="REDIS_URL" fi From 0216339b3e46cad58a5a9c30127229522beaff21 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Fri, 10 Feb 2023 09:34:29 +0100 Subject: [PATCH 171/175] More suggestions from documentation Co-authored-by: Bryce Eadie --- extra/datadog.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index 9d609c3..20b4885 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -177,7 +177,7 @@ export DD_PYTHONPATH="$DD_DIR/embedded/lib:$DD_PYTHONPATH" # Update the Postgres configuration from above using the Heroku application environment variable if [ "$ENABLE_HEROKU_POSTGRES" == "true" ]; then - # The default connection URL is DATABASE_URL, but can be configured by the user + # The default connection URL is set in DATABASE_URL, but can be configured by the user if [[ -z ${POSTGRES_URL_VAR} ]]; then POSTGRES_URL_VAR="DATABASE_URL" fi @@ -200,7 +200,7 @@ fi # Update the Redis configuration from above using the Heroku application environment variable if [ "$ENABLE_HEROKU_REDIS" == "true" ]; then - # The default connection URL is REDIS_URL, but can be configured by the user + # The default connection URL is set in REDIS_URL, but can be configured by the user if [[ -z ${REDIS_URL_VAR} ]]; then REDIS_URL_VAR="REDIS_URL" fi From 1e467233b14b5091f43481ee215f7e04c4e8f9be Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Mon, 27 Feb 2023 12:21:48 +0100 Subject: [PATCH 172/175] Pin versions 6.43 and 7.43 of the agent --- bin/compile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/compile b/bin/compile index 870dbfb..04866cb 100644 --- a/bin/compile +++ b/bin/compile @@ -11,8 +11,8 @@ set -o pipefail # set -x # Set agent pinned version -DD_AGENT_PINNED_VERSION_6="6.42.0-1" -DD_AGENT_PINNED_VERSION_7="7.42.0-1" +DD_AGENT_PINNED_VERSION_6="6.43.0-1" +DD_AGENT_PINNED_VERSION_7="7.43.0-1" # Parse and derive params BUILD_DIR=$1 From cd977684bf7cda50fb09dee8fd37d34679c43c62 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Mon, 27 Feb 2023 12:44:49 +0100 Subject: [PATCH 173/175] Prepare release 2.9 --- CHANGELOG.md | 9 +++++++++ extra/datadog.sh | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8803d28..1ba1c5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,15 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [2.9] - 2023-02-27 + +### Added +- Datadog agent pinned versions are now `6.43.0` and `7.43.0` +- Redis and Postgres integrations can be now configured setting an environment variable, without using the prerun.sh script. + +### Deprecated +- This is the last buildpack version with Datadog Agent 6.x as default. Upgrade today to Datadog Agent 7.x by setting the environment variable `DD_AGENT_MAJOR_VERSION=7` in your Heroku application and rebuilding the slug. + ## [2.8] - 2023-01-25 ### Removed diff --git a/extra/datadog.sh b/extra/datadog.sh index 20b4885..ed5baa7 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -73,7 +73,7 @@ done # Add tags to the config file DYNOHOST="$(hostname )" DYNOTYPE=${DYNO%%.*} -BUILDPACKVERSION="dev" +BUILDPACKVERSION="2.9" DYNO_TAGS="dyno:$DYNO dynotype:$DYNOTYPE buildpackversion:$BUILDPACKVERSION" # We want always to have the Dyno ID as a host alias to improve correlation From 21dd158da24f43d34d97a9628f0c4b50e6bfe2d2 Mon Sep 17 00:00:00 2001 From: Chris Wong Date: Mon, 29 Apr 2019 15:16:32 -0700 Subject: [PATCH 174/175] Rebase onto 2.9 from upstream. Allow the app to extend both the YAML and the environmental variables --- extra/datadog.sh | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/extra/datadog.sh b/extra/datadog.sh index ed5baa7..8b311b7 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -50,6 +50,14 @@ for dir in "$APP_DATADOG_CONF_DIR"/*; do done # Agent integrations configuration - Deprecated + +# Datadog.conf yaml addition (use sparingly, since parsing YAML is difficult) +APP_DATADOG_CONF_YAML_ADDON="/app/datadog/datadog-append.yaml" + +# Shell script to run additional functionality, as well as allowing +# environmental variables to be changed +APP_DATADOG_EXTENSION="/app/datadog/startup.sh" + for file in "$APP_DATADOG_CONF_DIR"/*.yaml; do test -f "$file" || continue # avoid errors when glob doesn't match anything filename="$(basename -- "$file")" @@ -104,6 +112,12 @@ sed -i -e"s|^logs_config:$|logs_config:\n run_path: $DD_RUN_DIR|" "$DATADOG_CON # For a list of env vars to override datadog.yaml, see: # https://github.com/DataDog/datadog-agent/blob/master/pkg/config/config.go#L145 + +# Read in files and extend $DATADOG_CONF with them if necessary +if [ -f ${APP_DATADOG_CONF_YAML_ADDON} ]; then + cat ${APP_DATADOG_CONF_YAML_ADDON} >> ${DATADOG_CONF} +fi + if [ -z "$DD_API_KEY" ]; then echo "DD_API_KEY environment variable not set. Run: heroku config:add DD_API_KEY=" DISABLE_DATADOG_AGENT=1 @@ -133,6 +147,12 @@ else fi fi +# Allow Apps to extend functionality, like disabling the Agent or changing +# environmental variables +if [ -f $APP_DATADOG_EXTENSION ]; then + . ${APP_DATADOG_EXTENSION} +fi + # Disable core checks (these read the host, not the dyno). if [ "$DD_DISABLE_HOST_METRICS" == "true" ]; then find "$DD_CONF_DIR"/conf.d -name "conf.yaml.default" -exec mv {} {}_disabled \; From 0ac1609fdc6c1d881eb22746afff462a52cf7773 Mon Sep 17 00:00:00 2001 From: ngtvspc Date: Thu, 20 Apr 2023 19:20:31 -0700 Subject: [PATCH 175/175] Remove reference to legacy startup.sh script --- extra/datadog.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/extra/datadog.sh b/extra/datadog.sh index 8b311b7..b5bd1be 100644 --- a/extra/datadog.sh +++ b/extra/datadog.sh @@ -54,10 +54,6 @@ done # Datadog.conf yaml addition (use sparingly, since parsing YAML is difficult) APP_DATADOG_CONF_YAML_ADDON="/app/datadog/datadog-append.yaml" -# Shell script to run additional functionality, as well as allowing -# environmental variables to be changed -APP_DATADOG_EXTENSION="/app/datadog/startup.sh" - for file in "$APP_DATADOG_CONF_DIR"/*.yaml; do test -f "$file" || continue # avoid errors when glob doesn't match anything filename="$(basename -- "$file")"