Turn on better logging for EC2 test framework (#2298)

andrewhop · web-flow · commit dc5fe8443ae3 · 2025-03-31T14:28:21.000-07:00
### Description of changes: 
Currently Cloudwatch logs only captures the output of commands run, not
which command ran so it's hard to figure out what is causing an issue.
This turns on logging for all commands, combines the standard error and
out logs, and turns on exit on any failure.

### Testing:
Before:
```
DEPENDENCIES_DIR=/home/dependencies
OLDPWD=/var/snap/amazon-ssm-agent/7984/aws-lc-pr
_=/usr/bin/env
Building AWS-LC in Release mode with address sanitizer, and running only non ssl test.
Using Ninja build system (ninja).
-- Go compiler 1.20.1 found
-- The C compiler identification is Clang 15.0.7
-- Detecting C compiler ABI info
-- Detecting C compiler ABI info - done
-- Check for working C compiler: /usr/bin/clang - skipped
-- Detecting C compile features
```

After:
```
+ echo 'Using Ninja build system (ninja).'
+ BUILD_COMMAND=ninja
+ cflags+=(-GNinja)
+ cmake3 -DASAN=1 -DCMAKE_BUILD_TYPE=Release -DENABLE_DILITHIUM=ON -GNinja /var/snap/amazon-ssm-agent/7984/aws-lc-pr
-- Go compiler 1.20.1 found
-- The C compiler identification is Clang 15.0.7
-- Detecting C compiler ABI info
-- Detecting C compiler ABI info - done
-- Check for working C compiler: /usr/bin/clang - skipped
-- Detecting C compile features
-- Detecting C compile features - done
```

By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache 2.0 license and the ISC license.
diff --git a/tests/ci/benchmark_framework/install_docker.sh b/tests/ci/benchmark_framework/install_docker.sh
@@ -2,7 +2,7 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0 OR ISC
 
-set -xo
+set -exo pipefail
 
 if ! [ -x "$(command -v docker)" ]; then
   apt-get update
diff --git a/tests/ci/cdk/cdk/ssm/general_test_run_ssm_document.yaml b/tests/ci/cdk/cdk/ssm/general_test_run_ssm_document.yaml
@@ -9,60 +9,65 @@ mainSteps:
     name: runShellScript
     inputs:
       timeoutSeconds: '7200'
+      shell: bash
       runCommand:
         # Fallback plan to shut down the ec2 instance in 90 minutes in case it's not terminated.
         # Codebuild just "stops" the instance calling the script, so "trap cleanup" is not executed.
-        - shutdown -P +90
-        - sudo -i
-        - systemctl stop apt-daily.timer
-        - export DEBIAN_FRONTEND=noninteractive
-        - export CPU_TYPE=$(dpkg --print-architecture)
-        - export SOURCE={SOURCE}
-        # if we have a cpu type of x86, we want linux-x86
-        - if [ "${CPU_TYPE}" = amd64 ]; then export CPU_ARCH=linux-x86; export AWS_CLI_PREFIX=x86_; fi
-        # if we have a cpu type of arm, we want linux-aarch
-        - if [ "${CPU_TYPE}" = arm64 ]; then export CPU_ARCH=linux-aarch; export AWS_CLI_PREFIX=aarch; fi
-        # install aws-cli
-        - killall apt apt-get
-        - apt-get update
-        - apt-get -y remove needrestart
-        - apt-get -y install unzip
-        - curl "https://awscli.amazonaws.com/awscli-exe-linux-${AWS_CLI_PREFIX}64.zip" -o "awscliv2.zip"
-        - unzip awscliv2.zip
-        - ./aws/install
-        # Check if the source code is on S3, otherwise treat the source as a PR.
-        - >
-          if [ "$(expr substr "$SOURCE" 1 16)" = "aws-lc-codebuild" ]; then
-            aws s3api get-object --bucket {S3_BUCKET} --key "${SOURCE##{S3_BUCKET}/}" aws-lc-pr.zip
-            unzip aws-lc-pr.zip
-          else
-            git clone {SOURCE} aws-lc-pr
-            cd aws-lc-pr
-            git fetch origin pull/{PR_NUM}/head:temp
-            git checkout temp
-            git show
-            if [ "$(git log -n 1 --pretty=format:"%H")" != "{COMMIT_ID}" ]; then
-              exit 1
+        - |
+            #!/usr/bin/env bash
+            # Redirect standard error to out so logs are combined in CloudWatch Logs
+            exec 2>&1
+            # Echo all commands before executing and exit on any failure
+            set -exo pipefail
+            shutdown -P +90
+            sudo -i
+            systemctl stop apt-daily.timer
+            export DEBIAN_FRONTEND=noninteractive
+            export CPU_TYPE=$(dpkg --print-architecture)
+            export SOURCE={SOURCE}
+            # if we have a cpu type of x86, we want linux-x86
+            if [ "${CPU_TYPE}" = amd64 ]; then export CPU_ARCH=linux-x86; export AWS_CLI_PREFIX=x86_; fi
+            # if we have a cpu type of arm, we want linux-aarch
+            if [ "${CPU_TYPE}" = arm64 ]; then export CPU_ARCH=linux-aarch; export AWS_CLI_PREFIX=aarch; fi
+            # install aws-cli
+            killall apt apt-get || echo "No apt processes found"
+            apt-get update
+            apt-get -y remove needrestart
+            apt-get -y install unzip
+            curl "https://awscli.amazonaws.com/awscli-exe-linux-${AWS_CLI_PREFIX}64.zip" -o "awscliv2.zip"
+            unzip awscliv2.zip
+            ./aws/install
+            # Check if the source code is on S3, otherwise treat the source as a PR.
+            if [ "$(expr substr "$SOURCE" 1 16)" = "aws-lc-codebuild" ]; then
+              aws s3api get-object --bucket {S3_BUCKET} --key "${SOURCE##{S3_BUCKET}/}" aws-lc-pr.zip
+              unzip aws-lc-pr.zip
+            else
+              git clone {SOURCE} aws-lc-pr
+              cd aws-lc-pr
+              git fetch origin pull/{PR_NUM}/head:temp
+              git checkout temp
+              git show
+              if [ "$(git log -n 1 --pretty=format:"%H")" != "{COMMIT_ID}" ]; then
+                exit 1
+              fi
             fi
-          fi
-        # install docker if its not already installed
-        - chmod +x ./tests/ci/benchmark_framework/install_docker.sh
-        - ./tests/ci/benchmark_framework/install_docker.sh
-        # log into docker and get needed docker image from ecr
-        - export ECR_REPO="{AWS_ACCOUNT_ID}.dkr.ecr.us-west-2.amazonaws.com/aws-lc-docker-images-${CPU_ARCH}"
-        - aws ecr get-login-password --region us-west-2 | docker login -u AWS --password-stdin "${ECR_REPO}"
-        - docker pull "${ECR_REPO}:{ECR_DOCKER_TAG}_latest"
-        # Start the container. Docker needs to be run in "privileged" mode for TSAN tests to pass.
-        - exec_docker="docker run -v `pwd`:`pwd` -w `pwd` --privileged ${ECR_REPO}:{ECR_DOCKER_TAG}_latest"
-        # Check if container was spun up succesfully. Then run test scripts and check the output.
-        - >
-          if [ -n "$exec_docker" ]; then
-            chmod +x {TARGET_TEST_SCRIPT}
-            $exec_docker {TARGET_TEST_SCRIPT}
-            if [ $? != 0 ]; then
+            # install docker if its not already installed
+            chmod +x ./tests/ci/benchmark_framework/install_docker.sh
+            ./tests/ci/benchmark_framework/install_docker.sh
+            # log into docker and get needed docker image from ecr
+            export ECR_REPO="{AWS_ACCOUNT_ID}.dkr.ecr.us-west-2.amazonaws.com/aws-lc-docker-images-${CPU_ARCH}"
+            aws ecr get-login-password --region us-west-2 | docker login -u AWS --password-stdin "${ECR_REPO}"
+            docker pull "${ECR_REPO}:{ECR_DOCKER_TAG}_latest"
+            # Start the container. Docker needs to be run in "privileged" mode for TSAN tests to pass.
+            exec_docker="docker run -v `pwd`:`pwd` -w `pwd` --privileged ${ECR_REPO}:{ECR_DOCKER_TAG}_latest"
+            # Check if container was spun up succesfully. Then run test scripts and check the output.
+            if [ -n "$exec_docker" ]; then
+              chmod +x {TARGET_TEST_SCRIPT}
+              $exec_docker {TARGET_TEST_SCRIPT}
+              if [ $? != 0 ]; then
+                exit 1
+              fi
+            else
               exit 1
             fi
-          else
-            exit 1
-          fi
-        - echo All ec2 test framework tests passed
+            echo All ec2 test framework tests passed
diff --git a/tests/ci/run_ec2_test_framework.sh b/tests/ci/run_ec2_test_framework.sh
@@ -95,8 +95,7 @@ ec2_test_ssm_command_id=$(run_ssm_command "${ssm_doc_name}" "${instance_id}" ${c
 run_url="https://${AWS_REGION}.console.aws.amazon.com/cloudwatch/home?region=${AWS_REGION}\
 #logsV2:log-groups/log-group/${cloudwatch_group_name}/log-events/${ec2_test_ssm_command_id}\$252F${instance_id}\$252FrunShellScript\$252F"
 
-echo "Actual Run in EC2 can be observered at CloudWatch URL: ${run_url}stdout"
-echo "Error outputs can be observered at CloudWatch URL: ${run_url}stderr"
+echo "Actual Run in EC2 can be observed at CloudWatch URL: ${run_url}stdout all output is redirected to stdout. If something is weird check the same url s/stdout/stderr"
 
 
 # Give some time for the commands to run, total wait time is 90 minutes.
