From 7265fa17a6bf3a48c86bf7464b0ba83d62685c1c Mon Sep 17 00:00:00 2001 From: Xinsong Cui Date: Tue, 23 Sep 2025 13:14:32 -0400 Subject: [PATCH 1/6] collect CI metric --- .github/actions/workflow-metrics/action.yml | 100 +++++++++++++++++++ .github/workflows/continuous-integration.yml | 29 +++++- 2 files changed, 127 insertions(+), 2 deletions(-) create mode 100644 .github/actions/workflow-metrics/action.yml diff --git a/.github/actions/workflow-metrics/action.yml b/.github/actions/workflow-metrics/action.yml new file mode 100644 index 000000000..8d21b99a0 --- /dev/null +++ b/.github/actions/workflow-metrics/action.yml @@ -0,0 +1,100 @@ +name: Workflow Metrics +description: > + Track and upload workflow metrics to CloudWatch + +runs: + using: composite + steps: + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + # will change this to AWS CI role before merging + role-to-assume: arn:aws:iam::886436966712:role/Admin + aws-region: us-west-2 + - name: Upload workflow metrics + shell: bash + run: | + # Determine build job name with matrix values + job_name="${{ github.job }}" + if [ ! -z "${{ matrix.java-version || '' }}" ]; then + job_name="${job_name}(${{ matrix.java-version }})" + fi + if [ ! -z "${{ matrix.os || '' }}" ]; then + job_name="${job_name}(${{ matrix.os }})" + fi + + # Determine success/failure (1 for success, 0 for failure) + if [ "${{ job.status }}" == "success" ]; then + success_value=1 + else + success_value=0 + fi + + # Determine branch (PR target branch or current branch) + if [ ! -z "${{ github.base_ref }}" ]; then + branch_name="${{ github.base_ref }}" + else + branch_name="${{ github.ref_name }}" + fi + + aws cloudwatch put-metric-data \ + --namespace "GitHub/Workflows" \ + --metric-data '[{ + "MetricName": "Success", + "Value": '$success_value', + "Unit": "Count", + "Dimensions": [ + { + "Name": "WorkflowName", + "Value": "${{ github.workflow }}" + }, + { + "Name": "JobName", + "Value": "'$job_name'" + }, + { + "Name": "Repository", + "Value": "${{ github.repository }}" + }, + { + "Name": "Branch", + "Value": "'$branch_name'" + } + ] + }]' + + if [ -z "$WORKFLOW_START_TIME" ]; then + echo "Warning: WORKFLOW_START_TIME not set, skipping metrics upload" + exit 0 + fi + + duration=$(($(date +%s) - $WORKFLOW_START_TIME)) + + # Only track duration for successful workflows + if [ "$success_value" -eq 1 ]; then + aws cloudwatch put-metric-data \ + --namespace "GitHub/Workflows" \ + --metric-data '[{ + "MetricName": "Duration", + "Value": '$duration', + "Unit": "Seconds", + "Dimensions": [ + { + "Name": "WorkflowName", + "Value": "${{ github.workflow }}" + }, + { + "Name": "JobName", + "Value": "'$job_name'" + }, + { + "Name": "Repository", + "Value": "${{ github.repository }}" + }, + { + "Name": "Branch", + "Value": "'$branch_name'" + } + ] + }]' + fi \ No newline at end of file diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 2a7c0cf3a..7e948e327 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -8,7 +8,9 @@ on: pull_request: workflow_dispatch: -permissions: { } +permissions: + id-token: write + contents: read # Allow one instance of this workflow per pull request, and cancel older runs when new changes are pushed concurrency: @@ -33,6 +35,8 @@ jobs: - 17 - 21 steps: + - name: Set start time + run: echo "WORKFLOW_START_TIME=$(date +%s)" >> $GITHUB_ENV - name: Checkout sources uses: actions/checkout@v4 - name: Configure JDK @@ -47,6 +51,9 @@ jobs: shell: bash run: | ./gradlew -Ptest.java.version=${{ matrix.java-version }} jvmTest --stacktrace + - name: Upload metrics + if: always() + uses: ./.github/actions/workflow-metrics all-platforms: runs-on: ${{ matrix.os }} @@ -55,6 +62,9 @@ jobs: matrix: os: [ ubuntu-latest, macos-latest, windows-latest ] steps: + - name: Set start time + shell: bash + run: echo "WORKFLOW_START_TIME=$(date +%s)" >> $GITHUB_ENV - name: Checkout sources uses: actions/checkout@v4 - name: Configure JDK @@ -71,6 +81,9 @@ jobs: echo "kotlinWarningsAsErrors=true" >> $GITHUB_WORKSPACE/local.properties ./gradlew apiCheck ./gradlew test jvmTest + - name: Upload metrics + if: always() + uses: ./.github/actions/workflow-metrics - name: Save Test Reports if: failure() uses: actions/upload-artifact@v4 @@ -81,6 +94,9 @@ jobs: protocol-tests: runs-on: ubuntu-latest steps: + - name: Set start time + shell: bash + run: echo "WORKFLOW_START_TIME=$(date +%s)" >> $GITHUB_ENV - name: Checkout sources uses: actions/checkout@v4 - name: Configure JDK @@ -96,10 +112,16 @@ jobs: run: | ./gradlew publishToMavenLocal ./gradlew testAllProtocols + - name: Upload metrics + if: always() + uses: ./.github/actions/workflow-metrics downstream: runs-on: ubuntu-latest steps: + - name: Set start time + shell: bash + run: echo "WORKFLOW_START_TIME=$(date +%s)" >> $GITHUB_ENV - name: Checkout sources uses: actions/checkout@v4 with: @@ -149,4 +171,7 @@ jobs: sed -i "s/smithy-kotlin-codegen-version = .*$/smithy-kotlin-codegen-version = \"$SMITHY_KOTLIN_CODEGEN_VERSION\"/" gradle/libs.versions.toml ./gradlew --parallel publishToMavenLocal ./gradlew test jvmTest - ./gradlew testAllProtocols \ No newline at end of file + ./gradlew testAllProtocols + - name: Upload metrics + if: always() + uses: ./smithy-kotlin/.github/actions/workflow-metrics \ No newline at end of file From d6e131a709e0afb09d927bc1cf2b802843ff2f96 Mon Sep 17 00:00:00 2001 From: Xinsong Cui Date: Tue, 23 Sep 2025 13:19:45 -0400 Subject: [PATCH 2/6] retrigger CI From 40459d6c066a846dcc7cadb77837b7b4c17a305b Mon Sep 17 00:00:00 2001 From: Xinsong Cui Date: Wed, 24 Sep 2025 16:53:02 -0400 Subject: [PATCH 3/6] use emit metric action --- .github/actions/workflow-metrics/action.yml | 100 -------------- .github/workflows/continuous-integration.yml | 135 ++++++++++++++++--- gradle/libs.versions.toml | 2 +- 3 files changed, 114 insertions(+), 123 deletions(-) delete mode 100644 .github/actions/workflow-metrics/action.yml diff --git a/.github/actions/workflow-metrics/action.yml b/.github/actions/workflow-metrics/action.yml deleted file mode 100644 index 8d21b99a0..000000000 --- a/.github/actions/workflow-metrics/action.yml +++ /dev/null @@ -1,100 +0,0 @@ -name: Workflow Metrics -description: > - Track and upload workflow metrics to CloudWatch - -runs: - using: composite - steps: - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - # will change this to AWS CI role before merging - role-to-assume: arn:aws:iam::886436966712:role/Admin - aws-region: us-west-2 - - name: Upload workflow metrics - shell: bash - run: | - # Determine build job name with matrix values - job_name="${{ github.job }}" - if [ ! -z "${{ matrix.java-version || '' }}" ]; then - job_name="${job_name}(${{ matrix.java-version }})" - fi - if [ ! -z "${{ matrix.os || '' }}" ]; then - job_name="${job_name}(${{ matrix.os }})" - fi - - # Determine success/failure (1 for success, 0 for failure) - if [ "${{ job.status }}" == "success" ]; then - success_value=1 - else - success_value=0 - fi - - # Determine branch (PR target branch or current branch) - if [ ! -z "${{ github.base_ref }}" ]; then - branch_name="${{ github.base_ref }}" - else - branch_name="${{ github.ref_name }}" - fi - - aws cloudwatch put-metric-data \ - --namespace "GitHub/Workflows" \ - --metric-data '[{ - "MetricName": "Success", - "Value": '$success_value', - "Unit": "Count", - "Dimensions": [ - { - "Name": "WorkflowName", - "Value": "${{ github.workflow }}" - }, - { - "Name": "JobName", - "Value": "'$job_name'" - }, - { - "Name": "Repository", - "Value": "${{ github.repository }}" - }, - { - "Name": "Branch", - "Value": "'$branch_name'" - } - ] - }]' - - if [ -z "$WORKFLOW_START_TIME" ]; then - echo "Warning: WORKFLOW_START_TIME not set, skipping metrics upload" - exit 0 - fi - - duration=$(($(date +%s) - $WORKFLOW_START_TIME)) - - # Only track duration for successful workflows - if [ "$success_value" -eq 1 ]; then - aws cloudwatch put-metric-data \ - --namespace "GitHub/Workflows" \ - --metric-data '[{ - "MetricName": "Duration", - "Value": '$duration', - "Unit": "Seconds", - "Dimensions": [ - { - "Name": "WorkflowName", - "Value": "${{ github.workflow }}" - }, - { - "Name": "JobName", - "Value": "'$job_name'" - }, - { - "Name": "Repository", - "Value": "${{ github.repository }}" - }, - { - "Name": "Branch", - "Value": "'$branch_name'" - } - ] - }]' - fi \ No newline at end of file diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 7e948e327..47999f497 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -35,8 +35,9 @@ jobs: - 17 - 21 steps: - - name: Set start time - run: echo "WORKFLOW_START_TIME=$(date +%s)" >> $GITHUB_ENV + - name: Set start timestamp + id: start + run: printf 'timestamp=%(%s)T\n' >> "$GITHUB_OUTPUT" - name: Checkout sources uses: actions/checkout@v4 - name: Configure JDK @@ -45,15 +46,37 @@ jobs: distribution: 'corretto' java-version: 17 cache: 'gradle' + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }} + aws-region: us-west-2 + - name: Setup kat + uses: aws/aws-kotlin-repo-tools/.github/actions/setup-kat@main - name: Configure Gradle uses: aws/aws-kotlin-repo-tools/.github/actions/configure-gradle@main - name: Test shell: bash run: | ./gradlew -Ptest.java.version=${{ matrix.java-version }} jvmTest --stacktrace - - name: Upload metrics - if: always() - uses: ./.github/actions/workflow-metrics + - name: Calculate duration + id: end + run: | + printf -v now '%(%s)T' + duration=$(( now - ${{ steps.start.outputs.timestamp }} )) + echo "duration=$duration" >> "$GITHUB_OUTPUT" + - name: Emit metrics + if: always() # run this step even if previous steps failed or the job is canceled + uses: aws/aws-kotlin-repo-tools/.github/actions/emit-metrics@main + with: + namespace: CI Metrics + dimensions: | + Product=aws-sdk-kotlin + JobName=${{ github.job }}(${{ matrix.java-version }}) + Branch=${{ github.base_ref || github.ref_name }} + metrics: | + WorkflowSucceeded:${{ job.status == 'success' && '1' || '0' }}:Count + WorkflowDuration:${{ steps.end.outputs.duration }}:Seconds all-platforms: runs-on: ${{ matrix.os }} @@ -62,9 +85,10 @@ jobs: matrix: os: [ ubuntu-latest, macos-latest, windows-latest ] steps: - - name: Set start time + - name: Set start timestamp + id: start shell: bash - run: echo "WORKFLOW_START_TIME=$(date +%s)" >> $GITHUB_ENV + run: echo "timestamp=$(date +%s)" >> "$GITHUB_OUTPUT" - name: Checkout sources uses: actions/checkout@v4 - name: Configure JDK @@ -73,6 +97,13 @@ jobs: distribution: 'corretto' java-version: 17 cache: 'gradle' + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }} + aws-region: us-west-2 + - name: Setup kat + uses: aws/aws-kotlin-repo-tools/.github/actions/setup-kat@main - name: Configure Gradle uses: aws/aws-kotlin-repo-tools/.github/actions/configure-gradle@main - name: Test @@ -81,9 +112,25 @@ jobs: echo "kotlinWarningsAsErrors=true" >> $GITHUB_WORKSPACE/local.properties ./gradlew apiCheck ./gradlew test jvmTest - - name: Upload metrics - if: always() - uses: ./.github/actions/workflow-metrics + - name: Calculate duration + id: end + shell: bash + run: | + now=$(date +%s) + duration=$(( now - ${{ steps.start.outputs.timestamp }} )) + echo "duration=$duration" >> "$GITHUB_OUTPUT" + - name: Emit metrics + if: always() # run this step even if previous steps failed or the job is canceled + uses: aws/aws-kotlin-repo-tools/.github/actions/emit-metrics@main + with: + namespace: CI Metrics + dimensions: | + Product=aws-sdk-kotlin + JobName=${{ github.job }}(${{ matrix.os }}) + Branch=${{ github.base_ref || github.ref_name }} + metrics: | + WorkflowSucceeded:${{ job.status == 'success' && '1' || '0' }}:Count + WorkflowDuration:${{ steps.end.outputs.duration }}:Seconds - name: Save Test Reports if: failure() uses: actions/upload-artifact@v4 @@ -94,9 +141,9 @@ jobs: protocol-tests: runs-on: ubuntu-latest steps: - - name: Set start time - shell: bash - run: echo "WORKFLOW_START_TIME=$(date +%s)" >> $GITHUB_ENV + - name: Set start timestamp + id: start + run: printf 'timestamp=%(%s)T\n' >> "$GITHUB_OUTPUT" - name: Checkout sources uses: actions/checkout@v4 - name: Configure JDK @@ -105,6 +152,13 @@ jobs: distribution: 'corretto' java-version: 17 cache: 'gradle' + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }} + aws-region: us-west-2 + - name: Setup kat + uses: aws/aws-kotlin-repo-tools/.github/actions/setup-kat@main - name: Configure Gradle uses: aws/aws-kotlin-repo-tools/.github/actions/configure-gradle@main - name: Test @@ -112,16 +166,31 @@ jobs: run: | ./gradlew publishToMavenLocal ./gradlew testAllProtocols - - name: Upload metrics - if: always() - uses: ./.github/actions/workflow-metrics + - name: Calculate duration + id: end + run: | + printf -v now '%(%s)T' + duration=$(( now - ${{ steps.start.outputs.timestamp }} )) + echo "duration=$duration" >> "$GITHUB_OUTPUT" + - name: Emit metrics + if: always() # run this step even if previous steps failed or the job is canceled + uses: aws/aws-kotlin-repo-tools/.github/actions/emit-metrics@main + with: + namespace: CI Metrics + dimensions: | + Product=aws-sdk-kotlin + JobName=${{ github.job }} + Branch=${{ github.base_ref || github.ref_name }} + metrics: | + WorkflowSucceeded:${{ job.status == 'success' && '1' || '0' }}:Count + WorkflowDuration:${{ steps.end.outputs.duration }}:Seconds downstream: runs-on: ubuntu-latest steps: - - name: Set start time - shell: bash - run: echo "WORKFLOW_START_TIME=$(date +%s)" >> $GITHUB_ENV + - name: Set start timestamp + id: start + run: printf 'timestamp=%(%s)T\n' >> "$GITHUB_OUTPUT" - name: Checkout sources uses: actions/checkout@v4 with: @@ -140,6 +209,13 @@ jobs: # smithy-kotlin is checked out as a sibling dir which will automatically make it an included build path: 'aws-sdk-kotlin' repository: 'aws/aws-sdk-kotlin' + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }} + aws-region: us-west-2 + - name: Setup kat + uses: aws/aws-kotlin-repo-tools/.github/actions/setup-kat@main - name: Configure Gradle - smithy-kotlin uses: aws/aws-kotlin-repo-tools/.github/actions/configure-gradle@main with: @@ -172,6 +248,21 @@ jobs: ./gradlew --parallel publishToMavenLocal ./gradlew test jvmTest ./gradlew testAllProtocols - - name: Upload metrics - if: always() - uses: ./smithy-kotlin/.github/actions/workflow-metrics \ No newline at end of file + - name: Calculate duration + id: end + run: | + printf -v now '%(%s)T' + duration=$(( now - ${{ steps.start.outputs.timestamp }} )) + echo "duration=$duration" >> "$GITHUB_OUTPUT" + - name: Emit metrics + if: always() # run this step even if previous steps failed or the job is canceled + uses: aws/aws-kotlin-repo-tools/.github/actions/emit-metrics@main + with: + namespace: CI Metrics + dimensions: | + Product=aws-sdk-kotlin + JobName=${{ github.job }} + Branch=${{ github.base_ref || github.ref_name }} + metrics: | + WorkflowSucceeded:${{ job.status == 'success' && '1' || '0' }}:Count + WorkflowDuration:${{ steps.end.outputs.duration }}:Seconds \ No newline at end of file diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 8b60cc05e..184a78daa 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -2,7 +2,7 @@ kotlin-version = "2.2.0" dokka-version = "2.0.0" -aws-kotlin-repo-tools-version = "0.4.56" +aws-kotlin-repo-tools-version = "0.4.58" # libs coroutines-version = "1.10.2" From 275da8f38dae5d6e4a64b1a17dc1a3480abe0943 Mon Sep 17 00:00:00 2001 From: Xinsong Cui Date: Wed, 24 Sep 2025 16:54:56 -0400 Subject: [PATCH 4/6] use correct product name --- .github/workflows/continuous-integration.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 47999f497..7becaf5d8 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -71,7 +71,7 @@ jobs: with: namespace: CI Metrics dimensions: | - Product=aws-sdk-kotlin + Product=smithy-kotlin JobName=${{ github.job }}(${{ matrix.java-version }}) Branch=${{ github.base_ref || github.ref_name }} metrics: | @@ -125,7 +125,7 @@ jobs: with: namespace: CI Metrics dimensions: | - Product=aws-sdk-kotlin + Product=smithy-kotlin JobName=${{ github.job }}(${{ matrix.os }}) Branch=${{ github.base_ref || github.ref_name }} metrics: | @@ -178,7 +178,7 @@ jobs: with: namespace: CI Metrics dimensions: | - Product=aws-sdk-kotlin + Product=smithy-kotlin JobName=${{ github.job }} Branch=${{ github.base_ref || github.ref_name }} metrics: | @@ -260,7 +260,7 @@ jobs: with: namespace: CI Metrics dimensions: | - Product=aws-sdk-kotlin + Product=smithy-kotlin JobName=${{ github.job }} Branch=${{ github.base_ref || github.ref_name }} metrics: | From 7f33b9704dac9f9521fdaa8ec946b6aae59b166f Mon Sep 17 00:00:00 2001 From: Xinsong Cui Date: Wed, 24 Sep 2025 17:23:17 -0400 Subject: [PATCH 5/6] format --- .github/workflows/continuous-integration.yml | 36 ++++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 7becaf5d8..598c20b63 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -248,21 +248,21 @@ jobs: ./gradlew --parallel publishToMavenLocal ./gradlew test jvmTest ./gradlew testAllProtocols - - name: Calculate duration - id: end - run: | - printf -v now '%(%s)T' - duration=$(( now - ${{ steps.start.outputs.timestamp }} )) - echo "duration=$duration" >> "$GITHUB_OUTPUT" - - name: Emit metrics - if: always() # run this step even if previous steps failed or the job is canceled - uses: aws/aws-kotlin-repo-tools/.github/actions/emit-metrics@main - with: - namespace: CI Metrics - dimensions: | - Product=smithy-kotlin - JobName=${{ github.job }} - Branch=${{ github.base_ref || github.ref_name }} - metrics: | - WorkflowSucceeded:${{ job.status == 'success' && '1' || '0' }}:Count - WorkflowDuration:${{ steps.end.outputs.duration }}:Seconds \ No newline at end of file + - name: Calculate duration + id: end + run: | + printf -v now '%(%s)T' + duration=$(( now - ${{ steps.start.outputs.timestamp }} )) + echo "duration=$duration" >> "$GITHUB_OUTPUT" + - name: Emit metrics + if: always() # run this step even if previous steps failed or the job is canceled + uses: aws/aws-kotlin-repo-tools/.github/actions/emit-metrics@main + with: + namespace: CI Metrics + dimensions: | + Product=smithy-kotlin + JobName=${{ github.job }} + Branch=${{ github.base_ref || github.ref_name }} + metrics: | + WorkflowSucceeded:${{ job.status == 'success' && '1' || '0' }}:Count + WorkflowDuration:${{ steps.end.outputs.duration }}:Seconds \ No newline at end of file From 737185952ababba87d6f6824293ad6ff1ad7813d Mon Sep 17 00:00:00 2001 From: Xinsong Cui Date: Wed, 24 Sep 2025 17:29:06 -0400 Subject: [PATCH 6/6] always calculate duration --- .github/workflows/continuous-integration.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 598c20b63..88cebb455 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -61,6 +61,7 @@ jobs: ./gradlew -Ptest.java.version=${{ matrix.java-version }} jvmTest --stacktrace - name: Calculate duration id: end + if: always() run: | printf -v now '%(%s)T' duration=$(( now - ${{ steps.start.outputs.timestamp }} )) @@ -114,6 +115,7 @@ jobs: ./gradlew test jvmTest - name: Calculate duration id: end + if: always() shell: bash run: | now=$(date +%s) @@ -168,6 +170,7 @@ jobs: ./gradlew testAllProtocols - name: Calculate duration id: end + if: always() run: | printf -v now '%(%s)T' duration=$(( now - ${{ steps.start.outputs.timestamp }} )) @@ -250,6 +253,7 @@ jobs: ./gradlew testAllProtocols - name: Calculate duration id: end + if: always() run: | printf -v now '%(%s)T' duration=$(( now - ${{ steps.start.outputs.timestamp }} ))