From af76fbba171714b224dd3a74d6984ba21c101c46 Mon Sep 17 00:00:00 2001 From: Soren Ptak Date: Tue, 15 Aug 2023 11:39:28 -0700 Subject: [PATCH 1/2] Changes to the coverage-cop to upload the HTML report --- coverage-cop/action.yml | 115 ++++++++++++++++++++++++++++++++++------ 1 file changed, 98 insertions(+), 17 deletions(-) diff --git a/coverage-cop/action.yml b/coverage-cop/action.yml index 1cdb584a..8588f2a4 100644 --- a/coverage-cop/action.yml +++ b/coverage-cop/action.yml @@ -1,42 +1,123 @@ name: 'coverage-cop' description: 'CI Check Coverage results of unit tests (using lcov)' inputs: - path: + coverage-file: description: 'Path to lcov output file containing coverage data.' required: true branch-coverage-min: description: 'The minumum required branch coverage (in %) for success' required: false - default: 100 + default: 95 line-coverage-min: description: 'The minumum required line coverage (in %) for success' required: false - default: 100 + default: 95 runs: using: "composite" - steps: - - name: Print coverage data - run: lcov --list --rc lcov_branch_coverage=1 ${{ inputs.path }} + steps: + - env: + stepName: Install Dependencies + bashPass: \033[32;1mPASSED - + bashInfo: \033[33;1mINFO - + bashFail: \033[31;1mFAILED - + bashEnd: \033[0 shell: bash - - name: Install lcov (if not present) - run: sudo apt-get install lcov + name: ${{ env.stepName }} + run: | + # ${{ env.stepName }} + echo "::group::${{ env.stepName }}" + sudo apt-get install lcov fd-find + echo "::endgroup::" + echo -e "${{ env.bashPass }} ${{env.stepName }} ${{ env.bashEnd }}" + + - env: + stepName: Check Line and Branch Coverage + bashPass: \033[32;1mPASSED - + bashInfo: \033[33;1mINFO - + bashFail: \033[31;1mFAILED - + bashEnd: \033[0 + name: ${{ env.stepName }} + id: action-check-line-and-branch-coverage shell: bash - - name: Check coverage run: | - LINE_COVERAGE=$(lcov --list ${{ inputs.path }} | tail -n 1 | cut -d '|' -f 2 | sed -n "s/\([^%]*\)%.*/\1/p") - BRANCH_COVERAGE=$(lcov --rc lcov_branch_coverage=1 --list ${{ inputs.path }} | tail -n 1 | cut -d '|' -f 4 | sed -n "s/\([^%]*\)%.*/\1/p") + # ${{ env.stepName }} + + # Print the received code cov report. + # TODO: The way it grabs the line/branch coverage is a little complicated + # I'd like to see if this can be done simpler. + echo -e " ${{ env.bashInfo }} Received LCov Report: ${{ inputs.line-coverage-min }} ${{ env.bashEnd}}" + lcov --list --rc lcov_branch_coverage=1 ${{ inputs.coverage-file }} + LINE_COVERAGE=$(lcov --list ${{ inputs.coverage-file }} | tail -n 1 | cut -d '|' -f 2 | sed -n "s/\([^%]*\)%.*/\1/p") + BRANCH_COVERAGE=$(lcov --rc lcov_branch_coverage=1 --list ${{ inputs.coverage-file }} | tail -n 1 | cut -d '|' -f 4 | sed -n "s/\([^%]*\)%.*/\1/p") RESULT=0 - echo "Required line coverage: ${{ inputs.line-coverage-min }}" - echo "Line coverage: $LINE_COVERAGE" + + # Check Line Coverage + echo -e " ${{ env.bashInfo }} Required Line Coverage: ${{ inputs.line-coverage-min }} ${{ env.bashEnd}}" + echo -e " ${{ env.bashInfo }} Received Line Coverage: $LINE_COVERAGE ${{ env.bashEnd}}" if [[ $(echo "$LINE_COVERAGE < ${{ inputs.line-coverage-min }}" | bc) -ne 0 ]]; then - echo "Line Coverage is too low." + echo -e "${{ env.bashFail }} Line Coverage is too low. ${{ env.bashEnd }}" RESULT=1 fi - echo "Required branch coverage: ${{ inputs.branch-coverage-min }}" - echo "Branch coverage: $BRANCH_COVERAGE" + + echo -e " ${{ env.bashInfo }} Required Branch Coverage: ${{ inputs.branch-coverage-min }} ${{ env.bashEnd}}" + echo -e " ${{ env.bashInfo }} Received Branch Coverage: $BRANCH_COVERAGE ${{ env.bashEnd}}" if [[ $(echo "$BRANCH_COVERAGE < ${{ inputs.branch-coverage-min }}" | bc) -ne 0 ]]; then - echo "Branch Coverage is too low." + echo -e "${{ env.bashFail }} Branch Coverage is too low. ${{ env.bashEnd }}" RESULT=1 fi + + if [ $RESULT -eq 0 ]; then + echo -e "${{ env.bashPass }} ${{ env.stepName }} ${{ env.bashEnd }}" + else + echo "::group::Create Failed Codecov HTML Report" + genhtml --rc lcov_branch_coverage=1 --ignore-errors source ${{ inputs.coverage-file }} --legend --title "$(basename `git rev-parse --show-toplevel`) $(git rev-parse HEAD)" --output-directory=CodecovHTMLReport + zip -r CodecovHTMLReport.zip CodecovHTMLReport + echo "::endgroup::" + echo -e "${{ env.bashFail }} ${{ env.stepName }} ${{ env.bashEnd }}" + fi exit $RESULT + + - name: Upload Failed Codecov HTML Report + if: failure() && ( steps.action-check-line-and-branch-coverage.outcome == 'failure' ) + id: upload-codecov-report + uses: actions/upload-artifact@v3 + with: + name: CodecovHTMLReport + path: CodecovHTMLReport.zip + retention-days: 5 + + - env: + stepName: Codecov Report Info + bashPass: \033[32;1m + bashInfo: \033[33;1m + bashFail: \033[31;1m + bashEnd: \033[0 + if: failure() && ( steps.upload-codecov-report.outcome == 'success' ) shell: bash + run: | + # ${{ env.stepName }} + echo -e "${{ env.bashInfo }} A zip file of the failed Codecov report has been attached to this workflow ${{ env.bashEnd }}" + echo -e "${{ env.bashInfo }} This can be accessed by returning to the bottom of the summary page of the workflow run ${{ env.bashEnd }}" + echo -e "${{ env.bashInfo }} At the bottom of the page will be a CodecovHTMLReport.zip file that you can download ${{ env.bashEnd }}" + echo -e "${{ env.bashInfo }} Unzip the file and then open the index.html file in your browser for more info missing branch and line coverage ${{ env.bashEnd }}" + exit 1 + + + +# We should use this - it creates a link on their website that you can access +# And it displays the results there, which is great. +# But it means we need to set up a Codecov dashboard for each repo and then +# Add that to a secret to use by default when using it. +# CorePKCS11 evidently has this, but I don't know who set that up or what approvals it took +# So For now I'm going to create the html report and add it to the run. +# More info here: https://about.Codecov.io/blog/how-to-set-up-Codecov-with-c-and-github-actions/ +# - env: +# stepName: Upload Line and Branch Report +# name: ${{ env.stepName }} +# if: failure() +# uses: Codecov/Codecov-action@v3 +# with: +# files: ${{ inputs.coverage-file }} +# flags: unit_tests +# fail_ci_if_error: false +# verbose: false From e329a8dc4f7b42af90f01327978c920aa90f9179 Mon Sep 17 00:00:00 2001 From: Soren Ptak Date: Wed, 30 Aug 2023 10:19:46 -0700 Subject: [PATCH 2/2] Use -e in the echo groups, update the test for coverage-cop --- .github/workflows/test.yml | 116 ++++++++++++++++++++++++++++++++----- coverage-cop/action.yml | 8 +-- 2 files changed, 104 insertions(+), 20 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 28b749b1..f404abeb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,6 +7,13 @@ on: branches: [main] workflow_dispatch: +env: + # The bash escape character is \033 + bashPass: \033[32;1mPASSED - + bashInfo: \033[33;1mINFO - + bashFail: \033[31;1mFAILED - + bashEnd: \033[0m + jobs: test-format-check: runs-on: ubuntu-20.04 @@ -109,7 +116,7 @@ jobs: exe-path: executable-monitor/test.out success-line: "SLEEPING FOR 6 SECONDS" timeout-seconds: 20 - + - name: Functional Test | Success Case | No Retries, No Success Line, Exit Code id: test-executable-monitor-action-no-retry-attempts-exit-code uses: ./executable-monitor @@ -125,7 +132,7 @@ jobs: exe-path: executable-monitor/test.out success-line: "SLEEPING FOR 6 SECONDS" retry-attempts: 2 - + - name: API Test | Success Case | Retries, No Success Line, Exit Code, Use Default Timeout id: test-executable-monitor-action-no-success-line-no-timeout uses: ./executable-monitor @@ -279,41 +286,118 @@ jobs: uses: ./spellings with: path: coreMQTT + test-coverage-cop: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + - uses: actions/checkout@v3 with: - repository: FreeRTOS/coreMQTT + repository: skptak/coreMQTT ref: main path: coreMQTT - - name: Build + + - env: + stepName: Build Core MQTT + name: ${{ env.stepName }} run: | + # ${{ env.stepName }} + exitStatus=0 + + echo -e "::group::Install Lcov" sudo apt-get install -y lcov + echo -e "::endgroup::" + + echo -e "::group::${{ env.stepName }}" cmake -S ./coreMQTT/test -B build/ \ -G "Unix Makefiles" \ -DCMAKE_BUILD_TYPE=Debug \ -DBUILD_CLONE_SUBMODULES=ON \ -DCMAKE_C_FLAGS='--coverage -Wall -Wextra -Werror -DNDEBUG -DLIBRARY_LOG_LEVEL=LOG_DEBUG' make -C build/ all - - name: Test - run: | - cd build/ - ctest -E system --output-on-failure - cd .. - - name: Run Coverage + exitStatus=$? + echo -e "::endgroup::" + + # Only get to here if we passed the build + echo -e "${{ env.bashPass }} ${{ env.stepName }} ${{ env.bashEnd }}" + + + - name: Run CTest for CoreMQTT + run: ctest --test-dir build -E system --output-on-failure + + - env: + stepName: Passing Coverage Test + shell: bash run: | + # ${{ env.stepName }} + echo -e "::group::Build Coverage for coreMQTT" make -C build/ coverage + echo -e "::endgroup::" + + echo -e "::group::${{ env.stepName }}" declare -a EXCLUDE=("\*test/\*" "\*CMakeCCompilerId\*" "\*mocks\*") - echo ${EXCLUDE[@]} | xargs lcov --rc lcov_branch_coverage=1 -r build/coverage.info -o build/coverage.info + echo ${EXCLUDE[@]} | xargs lcov --rc lcov_branch_coverage=1 --remove build/coverage.info -o build/coverage.info + echo -e "::endgroup::" + lcov --rc lcov_branch_coverage=1 --list build/coverage.info - - name: Test coverage cop action + echo -e "${{ env.bashPass }} ${{env.stepName }} ${{ env.bashEnd }}" + + - name: Test Coverage Cop Action + id: success-coverage-cop uses: ./coverage-cop with: - path: ./build/coverage.info + coverage-file: ./build/coverage.info branch-coverage-min: 70 line-coverage-min: 100 + + - env: + stepName: Create Lower Branch and Line Coverage Report + id: failure-coverage-action + continue-on-error: true + shell: bash + run: | + # ${{ env.stepName }} + echo -e "::group::Build Coverage for coreMQTT" + make -C build/ clean + make -C build/ coverage + echo -e "::endgroup::" + + echo -e "::group::${{ env.stepName }}" + declare -a EXCLUDE=("\*test/\*") + set +e + lcov --rc lcov_branch_coverage=1 --remove build/coverage.info -o build/coverage.info + echo -e "::endgroup::" + + lcov --rc lcov_branch_coverage=1 --list build/coverage.info + set -e + echo -e "${{ env.bashPass }} ${{env.stepName }} ${{ env.bashEnd }}" + + - name: Functional | Failure | Coverage Cop Failure Test Case + id: failure-coverage-cop-action + continue-on-error: true + uses: ./coverage-cop + with: + coverage-file: ./build/coverage.info + branch-coverage-min: 100 + line-coverage-min: 100 + + - env: + stepName: Check Failure Test Case + name: ${{ env.stepName }} + if: success() || failure() + id: check-failure-test-cases + shell: bash + run: | + # ${{ env.stepName }} + exitStatus=0 + if [ "${{ steps.failure-coverage-cop-action.outcome }}" = "failure" ]; then + echo -e "${{ env.bashPass }} Functional | Failure | Coverage Cop Failure Test Case | Had Expected "failure" ${{ env.bashEnd }}" + else + echo -e "${{ env.bashFail }} Functional | Failure | Coverage Cop Failure Test Case | Had Unexpected "success" ${{ env.bashEnd }}" + exitStatus=1 + fi + exit $exitStatus + test-memory-statistics: runs-on: ubuntu-latest steps: @@ -353,7 +437,7 @@ jobs: repository: FreeRTOS/FreeRTOS ref: '202107.00' path: FreeRTOS - submodules: recursive + submodules: recursive - name: Test manifest verifier uses: ./manifest-verifier with: diff --git a/coverage-cop/action.yml b/coverage-cop/action.yml index 8588f2a4..4b21596b 100644 --- a/coverage-cop/action.yml +++ b/coverage-cop/action.yml @@ -25,9 +25,9 @@ runs: name: ${{ env.stepName }} run: | # ${{ env.stepName }} - echo "::group::${{ env.stepName }}" + echo -e "::group::${{ env.stepName }}" sudo apt-get install lcov fd-find - echo "::endgroup::" + echo -e "::endgroup::" echo -e "${{ env.bashPass }} ${{env.stepName }} ${{ env.bashEnd }}" - env: @@ -69,10 +69,10 @@ runs: if [ $RESULT -eq 0 ]; then echo -e "${{ env.bashPass }} ${{ env.stepName }} ${{ env.bashEnd }}" else - echo "::group::Create Failed Codecov HTML Report" + echo -e "::group::Create Failed Codecov HTML Report" genhtml --rc lcov_branch_coverage=1 --ignore-errors source ${{ inputs.coverage-file }} --legend --title "$(basename `git rev-parse --show-toplevel`) $(git rev-parse HEAD)" --output-directory=CodecovHTMLReport zip -r CodecovHTMLReport.zip CodecovHTMLReport - echo "::endgroup::" + echo -e "::endgroup::" echo -e "${{ env.bashFail }} ${{ env.stepName }} ${{ env.bashEnd }}" fi exit $RESULT