From 86a82dbf0493bde5449fcfa8b67c1ef224f0788c Mon Sep 17 00:00:00 2001 From: Luca Burelli Date: Thu, 2 Oct 2025 19:30:52 +0200 Subject: [PATCH 1/5] ci/package_core: refactor to prepare platform independence Rename BOARD_VARIANTS to ALL_BOARD_DATA to reflect that it contains more than just board variants. Define PLAT environment variable so that it can be easily changed to build other platforms than arduino:zephyr. Signed-off-by: Luca Burelli --- .github/workflows/package_core.yml | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/.github/workflows/package_core.yml b/.github/workflows/package_core.yml index 887a67d8..6cd5ca58 100644 --- a/.github/workflows/package_core.yml +++ b/.github/workflows/package_core.yml @@ -14,7 +14,7 @@ jobs: outputs: CORE_TAG: ${{ env.CORE_TAG }} CORE_ARTIFACT: ${{ env.CORE_ARTIFACT }} - BOARD_VARIANTS: ${{ env.BOARD_VARIANTS }} + ALL_BOARD_DATA: ${{ env.ALL_BOARD_DATA }} steps: - name: Install OS dependencies working-directory: /opt @@ -34,7 +34,7 @@ jobs: yes | ./extra/bootstrap.sh -o=--filter=tree:0 echo "CORE_TAG=$(git describe --tags --exact-match || git describe --always)" >> "$GITHUB_ENV" echo "CORE_ARTIFACT=ArduinoCore-zephyr-$(git describe --always)" >> "$GITHUB_ENV" - echo "BOARD_VARIANTS=$(extra/get_board_details.sh | jq -cr 'sort_by(.variant)')" >> "$GITHUB_ENV" + echo "ALL_BOARD_DATA=$(extra/get_board_details.sh | jq -cr 'sort_by(.variant)')" >> "$GITHUB_ENV" - name: ccache uses: hendrikmuhs/ccache-action@v1.2 @@ -48,7 +48,7 @@ jobs: - name: Package core run: | - ./extra/package_core.sh ${{ env.CORE_TAG }} distrib/${{ env.CORE_ARTIFACT }}.tar.bz2 + ./extra/package_core.sh ${CORE_TAG} distrib/${CORE_ARTIFACT}.tar.bz2 - name: Archive core uses: actions/upload-artifact@v4 @@ -63,19 +63,21 @@ jobs: strategy: matrix: include: - ${{ fromJSON( needs.package-core.outputs.BOARD_VARIANTS ) }} + ${{ fromJSON( needs.package-core.outputs.ALL_BOARD_DATA ) }} fail-fast: false env: + PLAT: arduino:zephyr FQBN: arduino:zephyr:${{ matrix.board }} - REPORT_FILE: arduino-zephyr-${{ matrix.board }}.json + CORE_ARTIFACT: ${{ needs.package-core.outputs.CORE_ARTIFACT }} steps: - uses: actions/download-artifact@v4 with: - name: ${{ needs.package-core.outputs.CORE_ARTIFACT }} + name: ${{ env.CORE_ARTIFACT }} - name: Set up core run: | - tar xf ${{ needs.package-core.outputs.CORE_ARTIFACT }}.tar.bz2 + tar xf ${CORE_ARTIFACT}.tar.bz2 + echo "REPORT_FILE=$(echo ${FQBN} | tr ':' '-').json" >> $GITHUB_ENV - name: Create Blink sketch run: | @@ -90,7 +92,7 @@ jobs: # Use Board Manager to install the latest release of Arduino Zephyr Boards to get the toolchain - name: "arduino:zephyr" source-url: "https://downloads.arduino.cc/packages/package_zephyr_index.json" - - name: "arduino:zephyr" + - name: ${{ env.PLAT }} source-path: "ArduinoCore-zephyr" sketch-paths: Blink cli-compile-flags: | @@ -137,7 +139,7 @@ jobs: - test-core if: ${{ !cancelled() && needs.package-core.result == 'success' }} env: - BOARD_VARIANTS: ${{ needs.package-core.outputs.BOARD_VARIANTS }} + ALL_BOARD_DATA: ${{ needs.package-core.outputs.ALL_BOARD_DATA }} steps: - uses: actions/download-artifact@v4 with: @@ -147,11 +149,11 @@ jobs: - run: | echo "### Core test results:" >> "$GITHUB_STEP_SUMMARY" - jq -c '.[]' <<< "$BOARD_VARIANTS" | while read -r BOARD_VARIANT; do - BOARD=$(echo $BOARD_VARIANT | jq -cr '.board') - VARIANT=$(echo $BOARD_VARIANT | jq -cr '.variant') + jq -c '.[]' <<< "$ALL_BOARD_DATA" | while read -r BOARD_DATA; do + BOARD=$(echo $BOARD_DATA | jq -cr '.board') + VARIANT=$(echo $BOARD_DATA | jq -cr '.variant') FQBN="arduino:zephyr:$BOARD" - REPORT_FILE="arduino-zephyr-$BOARD.json" + REPORT_FILE="$(echo $FQBN | tr ':' '-').json" if [ ! -f $REPORT_FILE ]; then echo "* :x: $BOARD (`$VARIANT`) - No report found?" >> "$GITHUB_STEP_SUMMARY" else @@ -221,7 +223,7 @@ jobs: aws-region: ${{ secrets.AWS_REGION }} - name: Upload artifact - run: aws s3 cp ${{ env.ARTIFACT_FILE }} s3://${{ secrets.S3_BUCKET }}/ + run: aws s3 cp ${ARTIFACT_FILE} s3://${{ secrets.S3_BUCKET }}/ publish-json: name: Publish json From 90751a1c5a7bf80772e4119c79778cf5fef792b8 Mon Sep 17 00:00:00 2001 From: Luca Burelli Date: Fri, 3 Oct 2025 19:18:38 +0200 Subject: [PATCH 2/5] ci/package_core: refactor to prepare multi-package * Generate CORE_ARTIFACT every step so that multi-package is possible via matrix iterations in the next commit. * Convert gen_package_index_json.sh to take parameters instead of environment variables. * Export the name of the package from get_board_details.sh. * Rework package_core.sh so that it is easily extendable to multiple packages. Signed-off-by: Luca Burelli --- .github/workflows/package_core.yml | 49 ++++++++++++++++++------------ extra/bootstrap.sh | 2 ++ extra/gen_package_index_json.sh | 21 +++++++++---- extra/get_board_details.sh | 22 ++++++++++---- extra/package_core.sh | 35 +++++++++++++-------- 5 files changed, 84 insertions(+), 45 deletions(-) diff --git a/.github/workflows/package_core.yml b/.github/workflows/package_core.yml index 6cd5ca58..6767b017 100644 --- a/.github/workflows/package_core.yml +++ b/.github/workflows/package_core.yml @@ -13,12 +13,13 @@ jobs: CCACHE_IGNOREOPTIONS: -specs=* outputs: CORE_TAG: ${{ env.CORE_TAG }} - CORE_ARTIFACT: ${{ env.CORE_ARTIFACT }} + CORE_HASH: ${{ env.CORE_HASH }} ALL_BOARD_DATA: ${{ env.ALL_BOARD_DATA }} steps: - name: Install OS dependencies working-directory: /opt run: | + sudo apt-get remove --purge man-db -y # skips the mandb triggers sudo apt-get update sudo apt-get install -y --no-install-recommends git cmake wget python3-pip ninja-build ccache @@ -32,9 +33,15 @@ jobs: - name: Initialize Zephyr environment run: | yes | ./extra/bootstrap.sh -o=--filter=tree:0 - echo "CORE_TAG=$(git describe --tags --exact-match || git describe --always)" >> "$GITHUB_ENV" - echo "CORE_ARTIFACT=ArduinoCore-zephyr-$(git describe --always)" >> "$GITHUB_ENV" - echo "ALL_BOARD_DATA=$(extra/get_board_details.sh | jq -cr 'sort_by(.variant)')" >> "$GITHUB_ENV" + echo "CORE_HASH=$(git describe --always)" >> "$GITHUB_ENV" + echo "ALL_BOARD_DATA=$(extra/get_board_details.sh | jq -c 'sort_by(.variant)')" >> "$GITHUB_ENV" + + - name: Map output packages + # needs the above env vars to be usable + run: | + echo "CORE_TAG=$(git describe --tags --exact-match 2>/dev/null || echo $CORE_HASH)" >> "$GITHUB_ENV" + echo "ARTIFACTS=$(jq -c '["zephyr"] + (map(.artifact) | unique)' <<< ${ALL_BOARD_DATA})" >> "$GITHUB_ENV" + echo "SUB_ARCHES=$(jq -c 'map(.subarch) | unique' <<< ${ALL_BOARD_DATA})" >> "$GITHUB_ENV" - name: ccache uses: hendrikmuhs/ccache-action@v1.2 @@ -48,18 +55,21 @@ jobs: - name: Package core run: | + PACKAGE=zephyr + CORE_ARTIFACT=ArduinoCore-$PACKAGE-${CORE_HASH} ./extra/package_core.sh ${CORE_TAG} distrib/${CORE_ARTIFACT}.tar.bz2 - name: Archive core uses: actions/upload-artifact@v4 with: - name: ${{ env.CORE_ARTIFACT }} - path: distrib/${{ env.CORE_ARTIFACT }}.tar.bz2 + name: ArduinoCore-zephyr-${{ env.CORE_HASH }} + path: distrib/*.tar.bz2 test-core: name: Test on ${{ matrix.board }} runs-on: ubuntu-latest - needs: package-core + needs: + - package-core strategy: matrix: include: @@ -68,7 +78,7 @@ jobs: env: PLAT: arduino:zephyr FQBN: arduino:zephyr:${{ matrix.board }} - CORE_ARTIFACT: ${{ needs.package-core.outputs.CORE_ARTIFACT }} + CORE_ARTIFACT: ArduinoCore-zephyr-${{ needs.package-core.outputs.CORE_HASH }} steps: - uses: actions/download-artifact@v4 with: @@ -76,7 +86,7 @@ jobs: - name: Set up core run: | - tar xf ${CORE_ARTIFACT}.tar.bz2 + tar xf ${CORE_ARTIFACT}.tar.bz2 # will create ArduinoCore-zephyr/ echo "REPORT_FILE=$(echo ${FQBN} | tr ':' '-').json" >> $GITHUB_ENV - name: Create Blink sketch @@ -209,8 +219,7 @@ jobs: id-token: write contents: read env: - CORE_ARTIFACT: ${{ needs.package-core.outputs.CORE_ARTIFACT }} - ARTIFACT_FILE: ${{ needs.package-core.outputs.CORE_ARTIFACT }}.tar.bz2 + CORE_ARTIFACT: ArduinoCore-zephyr-${{ needs.package-core.outputs.CORE_HASH }} steps: - uses: actions/download-artifact@v4 with: @@ -223,7 +232,7 @@ jobs: aws-region: ${{ secrets.AWS_REGION }} - name: Upload artifact - run: aws s3 cp ${ARTIFACT_FILE} s3://${{ secrets.S3_BUCKET }}/ + run: aws s3 cp ArduinoCore-*.tar.bz2 s3://${{ secrets.S3_BUCKET }}/ publish-json: name: Publish json @@ -232,12 +241,9 @@ jobs: needs: - package-core - test-core - - publish-core env: - CORE_ARTIFACT: ${{ needs.package-core.outputs.CORE_ARTIFACT }} - ARTIFACT_FILE: ${{ needs.package-core.outputs.CORE_ARTIFACT }}.tar.bz2 + CORE_ARTIFACT: ArduinoCore-zephyr-${{ needs.package-core.outputs.CORE_HASH }} CORE_TAG: ${{ needs.package-core.outputs.CORE_TAG }} - PACKAGE_INDEX_JSON: zephyr-core-${{ needs.package-core.outputs.CORE_TAG }}.json steps: - uses: actions/checkout@v4 with: @@ -251,12 +257,15 @@ jobs: with: name: ${{ env.CORE_ARTIFACT }} - # uses: ARTIFACT_FILE CORE_TAG PACKAGE_INDEX_JSON - name: Prepare package index snippet - run: ./extra/gen_package_index_json.sh + run: | + PACKAGE=zephyr + ARTIFACT_FILE=ArduinoCore-${PACKAGE}-${CORE_HASH}.tar.bz2 + PACKAGE_JSON=ArduinoCore-${PACKAGE}-${CORE_TAG}.json + ./extra/gen_package_index_json.sh ${CORE_TAG} ${ARTIFACT_FILE} > ${PACKAGE_JSON} - name: Archive package index snippet uses: actions/upload-artifact@v4 with: - name: ${{ env.PACKAGE_INDEX_JSON }} - path: ${{ env.PACKAGE_INDEX_JSON }} + name: ArduinoCore-zephyr-${{ env.CORE_TAG }}-jsons + path: ArduinoCore-*-${{ env.CORE_TAG }}.json diff --git a/extra/bootstrap.sh b/extra/bootstrap.sh index 99c7f3cc..bf63e65d 100755 --- a/extra/bootstrap.sh +++ b/extra/bootstrap.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -e + if [ ! -f platform.txt ]; then echo Launch this script from the root core folder as ./extra/bootstrap.sh exit 2 diff --git a/extra/gen_package_index_json.sh b/extra/gen_package_index_json.sh index 6cb0eb10..2e3c4e87 100755 --- a/extra/gen_package_index_json.sh +++ b/extra/gen_package_index_json.sh @@ -1,13 +1,22 @@ #!/bin/bash -if [ -z "$CORE_TAG" ]; then - echo "This script can be used in Github CI only." +if [ $# -ne 2 ] ; then + echo "Usage: $0 " exit 1 fi -cat extra/zephyr-core-template.json | sed \ +PACKAGE=zephyr +CORE_TAG=$1 +ARTIFACT_FILE=$2 + +if [ ! -f "$ARTIFACT_FILE" ] ; then + echo "Artifact file '$ARTIFACT_FILE' not found" 1>&2 + exit 1 +fi + +JSON_TEMPLATE="extra/zephyr-core-template.json" +cat $JSON_TEMPLATE | sed \ -e "s/__CORE_TAG__/$CORE_TAG/" \ - -e "s/__ARTIFACT_FILE__/$ARTIFACT_FILE/" \ + -e "s/__ARTIFACT_FILE__/$(basename $ARTIFACT_FILE)/" \ -e "s/__ARTIFACT_HASH__/$(sha256sum $ARTIFACT_FILE | awk '{print $1}')/" \ - -e "s/__ARTIFACT_SIZE__/$(stat -c %s $ARTIFACT_FILE)/" \ - > $PACKAGE_INDEX_JSON + -e "s/__ARTIFACT_SIZE__/$(stat -c %s $ARTIFACT_FILE)/" | jq . diff --git a/extra/get_board_details.sh b/extra/get_board_details.sh index 035b101d..a92b850d 100755 --- a/extra/get_board_details.sh +++ b/extra/get_board_details.sh @@ -7,14 +7,15 @@ get_boards() { get_board_field() { board=$1 field=$2 - cat boards.txt | sed -e 's/#.*//' | grep -E "^$board\\.build\\.$field=" | cut -d '=' -f2- | sed -e 's/"/\"/g' + cat boards.txt | sed -e 's/#.*//' | grep -E "^$board\\.$field=" | cut -d '=' -f2- | sed -e 's/"/\"/g' } for BOARD in $(get_boards); do - VARIANT=$(get_board_field $BOARD "variant") - TARGET=$(get_board_field $BOARD "zephyr_target") - ARGS=$(get_board_field $BOARD "zephyr_args") - HALS=$(get_board_field $BOARD "zephyr_hals") + NAME=$(get_board_field $BOARD "name") + VARIANT=$(get_board_field $BOARD "build\\.variant") + TARGET=$(get_board_field $BOARD "build\\.zephyr_target") + ARGS=$(get_board_field $BOARD "build\\.zephyr_args") + HALS=$(get_board_field $BOARD "build\\.zephyr_hals") if [ -z "$TARGET" ] ; then echo "error: missing '$BOARD.build.zephyr_target'" 1>&2 @@ -25,5 +26,14 @@ for BOARD in $(get_boards); do exit 1 fi - echo "{ \"board\": \"$BOARD\", \"variant\": \"$VARIANT\", \"target\": \"$TARGET\", \"args\": \"$ARGS\", \"hals\": \"$HALS\" }" + cat << EOF + { + "name": "$NAME", + "board": "$BOARD", + "variant": "$VARIANT", + "target": "$TARGET", + "args": "$ARGS", + "hals": "$HALS", + } +EOF done | jq -crs . diff --git a/extra/package_core.sh b/extra/package_core.sh index 3b0b46ae..ff4c0a35 100755 --- a/extra/package_core.sh +++ b/extra/package_core.sh @@ -8,11 +8,11 @@ if [ -z "$1" ]; then fi if [ ! -f platform.txt ]; then - echo "Launch this script from the root core folder as ./extra/package_core.sh VERSION" + echo "Launch this script from the root core folder." exit 2 fi -PACKAGE=ArduinoCore-zephyr +PACKAGE=zephyr VERSION=$1 OUTPUT_FILE=${2:-distrib/${PACKAGE}-${VERSION}.tar.bz2} @@ -20,19 +20,28 @@ OUTPUT_FILE=${2:-distrib/${PACKAGE}-${VERSION}.tar.bz2} TEMP_PLATFORM=$(mktemp -p . | sed 's/\.\///') sed -e "s/^version=.*/version=${VERSION}/" platform.txt > ${TEMP_PLATFORM} -TEMP_LIST=$(mktemp) -echo ${TEMP_PLATFORM} > ${TEMP_LIST} +declutter_file() { + [ -f "$1" ] || return 0 + cat "$1" | sed -e 's/\s*#.*//' | grep -v '^\s*$' +} -# import a basic list of files and directories -cat extra/package_core.inc | sed -e 's/\s*#.*//' | grep -v '^\s*$' >> ${TEMP_LIST} - -# add the board-specific files +# create the list of files and directories to include +TEMP_INC=$(mktemp -p . | sed 's/\.\///') +echo ${TEMP_PLATFORM} >> ${TEMP_INC} +declutter_file extra/package_core.inc >> ${TEMP_INC} extra/get_board_details.sh | jq -cr '.[]' | while read -r item; do variant=$(jq -cr '.variant' <<< "$item") - echo "variants/${variant}/" >> ${TEMP_LIST} - ls firmwares/zephyr-${variant}.* >> ${TEMP_LIST} + echo "::info::`${variant}`" + echo "variants/${variant}/" >> ${TEMP_INC} + ls firmwares/zephyr-${variant}.* >> ${TEMP_INC} done -cat ${TEMP_LIST} + +# create the list of files and directories to exclude +TEMP_EXC=$(mktemp -p . | sed 's/\.\///') +declutter_file extra/package.exc >> ${TEMP_EXC} + mkdir -p $(dirname ${OUTPUT_FILE}) -tar -cjhf ${OUTPUT_FILE} -X extra/package_core.exc -T ${TEMP_LIST} --transform "s,${TEMP_PLATFORM},platform.txt," --transform "s,^,${PACKAGE}/," -rm -f ${TEMP_LIST} ${TEMP_PLATFORM} +tar -cjhf ${OUTPUT_FILE} -X ${TEMP_EXC} -T ${TEMP_INC} \ + --transform "s,${TEMP_PLATFORM},platform.txt," \ + --transform "s,^,ArduinoCore-zephyr/," +rm -f ${TEMP_INC} ${TEMP_EXC} ${TEMP_PLATFORM} From 6617195c751f6b7c00954d2201eb4eeb3534f1fc Mon Sep 17 00:00:00 2001 From: Luca Burelli Date: Sun, 12 Oct 2025 18:12:03 +0200 Subject: [PATCH 3/5] ci/package_core: generate SemVer-compatible core version Add the get_core_version.sh script to generate a SemVer-compatible version number for the core, based on Git tags. Update package_core.sh and gen_package_index_json.sh to use it. Signed-off-by: Luca Burelli --- .github/workflows/package_core.yml | 1 + extra/gen_package_index_json.sh | 2 +- extra/get_core_version.sh | 38 ++++++++++++++++++++++++++++++ extra/package_core.sh | 3 ++- 4 files changed, 42 insertions(+), 2 deletions(-) create mode 100755 extra/get_core_version.sh diff --git a/.github/workflows/package_core.yml b/.github/workflows/package_core.yml index 6767b017..4051564e 100644 --- a/.github/workflows/package_core.yml +++ b/.github/workflows/package_core.yml @@ -35,6 +35,7 @@ jobs: yes | ./extra/bootstrap.sh -o=--filter=tree:0 echo "CORE_HASH=$(git describe --always)" >> "$GITHUB_ENV" echo "ALL_BOARD_DATA=$(extra/get_board_details.sh | jq -c 'sort_by(.variant)')" >> "$GITHUB_ENV" + echo "## Building \`$(extra/get_core_version.sh)\`" >> "$GITHUB_STEP_SUMMARY" - name: Map output packages # needs the above env vars to be usable diff --git a/extra/gen_package_index_json.sh b/extra/gen_package_index_json.sh index 2e3c4e87..d86df9bf 100755 --- a/extra/gen_package_index_json.sh +++ b/extra/gen_package_index_json.sh @@ -16,7 +16,7 @@ fi JSON_TEMPLATE="extra/zephyr-core-template.json" cat $JSON_TEMPLATE | sed \ - -e "s/__CORE_TAG__/$CORE_TAG/" \ + -e "s/__CORE_TAG__/$(extra/get_core_version.sh)/" \ -e "s/__ARTIFACT_FILE__/$(basename $ARTIFACT_FILE)/" \ -e "s/__ARTIFACT_HASH__/$(sha256sum $ARTIFACT_FILE | awk '{print $1}')/" \ -e "s/__ARTIFACT_SIZE__/$(stat -c %s $ARTIFACT_FILE)/" | jq . diff --git a/extra/get_core_version.sh b/extra/get_core_version.sh new file mode 100755 index 00000000..5cfdf6b7 --- /dev/null +++ b/extra/get_core_version.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# This script generates a SemVer-compatible version number based on Git tags. +# +# If the current commit is tagged, it returns that version. If not, it +# generates a version string based on the next patch number and the current +# commit hash. +# +# If the tag is a simple "..", git describe will output: +# +# ..--g +# +# The regexp splits the patch number, awk converts the tokens to: +# +# ..-0.dev+ +# +# If the tag refers to a pre-release, like "..-", +# sed will output a total of 4 arguments and the format of the final awk output +# will be: +# +# ..--0.dev+ +# +# These are among the lowest possible SemVer versions greater than the last +# tagged version. +# +# If there are no tags at all (for example when run in a fork etc), it defaults +# to "9.9.9-+". + +VERSION=$(git describe --tags --exact-match 2>/dev/null) +if [ -z "$VERSION" ]; then + VERSION=$(git describe --tags --dirty 2>/dev/null | + sed 's/\.\([[:digit:]]\+\)\(-.*\)*-[[:digit:]]\+-g/ \1 \2 /' | + awk '{ if (NF==3) { print $1 "." ($2+1) "-0.dev+" $3 } else { print $1 "." $2 $3 "-0.dev+" $4 }}') + if [ -z "$VERSION" ]; then + VERSION="9.9.9-$(date '+%Y%m%d-%H%M%S')+$(git describe --always --dirty)" + fi +fi +echo $VERSION diff --git a/extra/package_core.sh b/extra/package_core.sh index ff4c0a35..9af06163 100755 --- a/extra/package_core.sh +++ b/extra/package_core.sh @@ -18,7 +18,8 @@ OUTPUT_FILE=${2:-distrib/${PACKAGE}-${VERSION}.tar.bz2} # create a temporary platform.txt file with the correct version TEMP_PLATFORM=$(mktemp -p . | sed 's/\.\///') -sed -e "s/^version=.*/version=${VERSION}/" platform.txt > ${TEMP_PLATFORM} +cat platform.txt > ${TEMP_PLATFORM} +sed -ie "s/^version=.*/version=$(extra/get_core_version.sh)/" ${TEMP_PLATFORM} declutter_file() { [ -f "$1" ] || return 0 From 439fb767aa7694195f4f6dc131cc3032564e7a3d Mon Sep 17 00:00:00 2001 From: Luca Burelli Date: Sun, 12 Oct 2025 18:36:33 +0200 Subject: [PATCH 4/5] ci/package_core: multi-artifact support Add a new "board.build.artifact" option to boards.txt to specify which artifact a board belongs to. If not specified, the board will automatically be added to "zephyr_contrib". The artifact names collected from boards.txt (plus the auto "zephyr" package, which includes all boards) are used to build and package each artifact in the GitHub Actions workflow; in particular, the FQBN of each board will depend on its artifact's "architecture" field. The test action is also updated to test each artifact on the appropriate boards. Individual package JSON files are in the extra/artifacts directory, along with common include/exclude files for packaging. Signed-off-by: Luca Burelli --- .github/workflows/package_core.yml | 149 +++++++++++------- boards.txt | 16 +- .../_common.exc} | 0 .../_common.inc} | 3 +- extra/artifacts/_common.json | 51 ++++++ extra/artifacts/zephyr_contrib.json | 5 + extra/artifacts/zephyr_main.json | 5 + extra/artifacts/zephyr_unoq.json | 5 + extra/build_all.sh | 9 +- extra/gen_package_index_json.sh | 42 +++-- extra/get_board_details.sh | 13 ++ extra/package_core.sh | 53 +++++-- extra/zephyr-core-template.json | 87 ---------- 13 files changed, 267 insertions(+), 171 deletions(-) rename extra/{package_core.exc => artifacts/_common.exc} (100%) rename extra/{package_core.inc => artifacts/_common.inc} (87%) create mode 100644 extra/artifacts/_common.json create mode 100644 extra/artifacts/zephyr_contrib.json create mode 100644 extra/artifacts/zephyr_main.json create mode 100644 extra/artifacts/zephyr_unoq.json delete mode 100644 extra/zephyr-core-template.json diff --git a/.github/workflows/package_core.yml b/.github/workflows/package_core.yml index 4051564e..ebf73fd3 100644 --- a/.github/workflows/package_core.yml +++ b/.github/workflows/package_core.yml @@ -7,7 +7,7 @@ on: jobs: package-core: - name: Build and package core + name: Build and package cores runs-on: ubuntu-latest env: CCACHE_IGNOREOPTIONS: -specs=* @@ -15,6 +15,8 @@ jobs: CORE_TAG: ${{ env.CORE_TAG }} CORE_HASH: ${{ env.CORE_HASH }} ALL_BOARD_DATA: ${{ env.ALL_BOARD_DATA }} + ARTIFACTS: ${{ env.ARTIFACTS }} + SUB_ARCHES: ${{ env.SUB_ARCHES }} steps: - name: Install OS dependencies working-directory: /opt @@ -54,32 +56,54 @@ jobs: run: | ./extra/build_all.sh -f - - name: Package core + - name: Package cores run: | - PACKAGE=zephyr - CORE_ARTIFACT=ArduinoCore-$PACKAGE-${CORE_HASH} - ./extra/package_core.sh ${CORE_TAG} distrib/${CORE_ARTIFACT}.tar.bz2 + jq -cr '.[]' <<< ${ARTIFACTS} | while read -r artifact; do + ARTIFACT_NAME=ArduinoCore-${artifact}-${CORE_HASH} + ./extra/package_core.sh ${artifact} ${CORE_TAG} distrib/${ARTIFACT_NAME}.tar.bz2 + done - - name: Archive core + - name: Archive cores uses: actions/upload-artifact@v4 with: - name: ArduinoCore-zephyr-${{ env.CORE_HASH }} + name: ArduinoCore-archives-${{ env.CORE_HASH }} path: distrib/*.tar.bz2 + split-core: + name: Split off ${{ matrix.artifact }} + runs-on: ubuntu-latest + needs: package-core + env: + ALL_CORES_ARTIFACT: ArduinoCore-archives-${{ needs.package-core.outputs.CORE_HASH }} + CORE_ARTIFACT: ArduinoCore-${{ matrix.artifact }}-${{ needs.package-core.outputs.CORE_HASH }} + strategy: + matrix: + artifact: ${{ fromJSON( needs.package-core.outputs.ARTIFACTS ) }} + steps: + - uses: actions/download-artifact@v4 + with: + name: ${{ env.ALL_CORES_ARTIFACT }} + + - uses: actions/upload-artifact@v4 + with: + name: ${{ env.CORE_ARTIFACT }} + path: ${{ env.CORE_ARTIFACT }}.tar.bz2 + test-core: - name: Test on ${{ matrix.board }} + name: Test ${{ matrix.subarch }}:${{ matrix.board }} runs-on: ubuntu-latest needs: - package-core + - split-core strategy: matrix: include: ${{ fromJSON( needs.package-core.outputs.ALL_BOARD_DATA ) }} fail-fast: false env: - PLAT: arduino:zephyr - FQBN: arduino:zephyr:${{ matrix.board }} - CORE_ARTIFACT: ArduinoCore-zephyr-${{ needs.package-core.outputs.CORE_HASH }} + PLAT: arduino:${{ matrix.subarch }} + FQBN: arduino:${{ matrix.subarch }}:${{ matrix.board }} + CORE_ARTIFACT: ArduinoCore-${{ matrix.artifact }}-${{ needs.package-core.outputs.CORE_HASH }} steps: - uses: actions/download-artifact@v4 with: @@ -127,7 +151,7 @@ jobs: repo: context.repo.repo, run_id: context.runId }); - const job_name = `Test on ${{ matrix.board }}` + const job_name = `Test ${{ matrix.subarch }}:${{ matrix.board }}` return workflow_run.jobs.find((job) => job.name === job_name).id; - name: Prepare log @@ -159,37 +183,41 @@ jobs: merge-multiple: true - run: | - echo "### Core test results:" >> "$GITHUB_STEP_SUMMARY" - jq -c '.[]' <<< "$ALL_BOARD_DATA" | while read -r BOARD_DATA; do - BOARD=$(echo $BOARD_DATA | jq -cr '.board') - VARIANT=$(echo $BOARD_DATA | jq -cr '.variant') - FQBN="arduino:zephyr:$BOARD" - REPORT_FILE="$(echo $FQBN | tr ':' '-').json" - if [ ! -f $REPORT_FILE ]; then - echo "* :x: $BOARD (`$VARIANT`) - No report found?" >> "$GITHUB_STEP_SUMMARY" - else - REPORT=$(jq -cr '.boards[0].sketches[0]' $REPORT_FILE) - JOB_ID=$(echo $REPORT | jq -cr '.job_id') - JOB_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/job/${JOB_ID}#step:5:2" - if ! $(echo $REPORT | jq -cr '.compilation_success') ; then - echo "* :x: [$BOARD]($JOB_URL) (\`$VARIANT\`) - Build failed" >> "$GITHUB_STEP_SUMMARY" + ARTIFACTS=$(jq -cr 'map(.artifact) | unique | .[]' <<< ${ALL_BOARD_DATA}) # this avoids the 'zephyr' artifact + for artifact in $ARTIFACTS ; do + echo "### \`$artifact\` test results:" >> "$GITHUB_STEP_SUMMARY" + jq -c "map(select(.artifact == \"$artifact\")) | .[]" <<< ${ALL_BOARD_DATA} | while read -r BOARD_DATA; do + BOARD=$(echo $BOARD_DATA | jq -cr '.board') + VARIANT=$(echo $BOARD_DATA | jq -cr '.variant') + SUBARCH=$(echo $BOARD_DATA | jq -cr '.subarch') + FQBN="arduino:$SUBARCH:$BOARD" + REPORT_FILE="$(echo $FQBN | tr ':' '-').json" + if [ ! -f $REPORT_FILE ]; then + echo "* :x: $BOARD (\`$VARIANT\`) - No report found?" >> "$GITHUB_STEP_SUMMARY" else - WARNINGS=$(echo $REPORT | jq -cr '.warnings.current.absolute // 0') - if [ $WARNINGS -eq 0 ]; then - echo "* :white_check_mark: $BOARD (\`$VARIANT\`) - Build successful" >> "$GITHUB_STEP_SUMMARY" + REPORT=$(jq -cr '.boards[0].sketches[0]' $REPORT_FILE) + JOB_ID=$(echo $REPORT | jq -cr '.job_id') + JOB_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/job/${JOB_ID}#step:5:2" + if ! $(echo $REPORT | jq -cr '.compilation_success') ; then + echo "* :x: [$BOARD]($JOB_URL) (\`$VARIANT\`) - Build failed" >> "$GITHUB_STEP_SUMMARY" else - echo >> "$GITHUB_STEP_SUMMARY" - echo "
   :warning: $BOARD ($VARIANT) - $WARNINGS warnings:" >> "$GITHUB_STEP_SUMMARY" - echo >> "$GITHUB_STEP_SUMMARY" - echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY" - echo $REPORT | jq -cr '.warnings_log[]' >> "$GITHUB_STEP_SUMMARY" - echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY" - echo >> "$GITHUB_STEP_SUMMARY" - echo "
" >> "$GITHUB_STEP_SUMMARY" - echo >> "$GITHUB_STEP_SUMMARY" + WARNINGS=$(echo $REPORT | jq -cr '.warnings.current.absolute // 0') + if [ $WARNINGS -eq 0 ]; then + echo "* :white_check_mark: $BOARD (\`$VARIANT\`) - Build successful" >> "$GITHUB_STEP_SUMMARY" + else + echo >> "$GITHUB_STEP_SUMMARY" + echo "
   :warning: $BOARD ($VARIANT) - $WARNINGS warnings:" >> "$GITHUB_STEP_SUMMARY" + echo >> "$GITHUB_STEP_SUMMARY" + echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY" + echo $REPORT | jq -cr '.warnings_log[]' >> "$GITHUB_STEP_SUMMARY" + echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY" + echo >> "$GITHUB_STEP_SUMMARY" + echo "
" >> "$GITHUB_STEP_SUMMARY" + echo >> "$GITHUB_STEP_SUMMARY" + fi fi fi - fi + done done - name: Clean up intermediate artifacts @@ -220,11 +248,11 @@ jobs: id-token: write contents: read env: - CORE_ARTIFACT: ArduinoCore-zephyr-${{ needs.package-core.outputs.CORE_HASH }} + ALL_CORES_ARTIFACT: ArduinoCore-archives-${{ needs.package-core.outputs.CORE_HASH }} steps: - uses: actions/download-artifact@v4 with: - name: ${{ env.CORE_ARTIFACT }} + name: ${{ env.ALL_CORES_ARTIFACT }} - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 @@ -236,37 +264,52 @@ jobs: run: aws s3 cp ArduinoCore-*.tar.bz2 s3://${{ secrets.S3_BUCKET }}/ publish-json: - name: Publish json + name: Publish jsons runs-on: ubuntu-latest if: ${{ github.event_name == 'push' && github.repository == 'arduino/ArduinoCore-zephyr' }} needs: - package-core - test-core env: - CORE_ARTIFACT: ArduinoCore-zephyr-${{ needs.package-core.outputs.CORE_HASH }} + ALL_CORES_ARTIFACT: ArduinoCore-archives-${{ needs.package-core.outputs.CORE_HASH }} CORE_TAG: ${{ needs.package-core.outputs.CORE_TAG }} + CORE_HASH: ${{ needs.package-core.outputs.CORE_HASH }} + ARTIFACTS: ${{ needs.package-core.outputs.ARTIFACTS }} steps: - uses: actions/checkout@v4 with: fetch-depth: 0 persist-credentials: false - sparse-checkout: | - extra/gen_package_index_json.sh - extra/zephyr-core-template.json + fetch-tags: true - uses: actions/download-artifact@v4 with: - name: ${{ env.CORE_ARTIFACT }} + name: ${{ env.ALL_CORES_ARTIFACT }} - - name: Prepare package index snippet + - name: Prepare package index snippets run: | - PACKAGE=zephyr - ARTIFACT_FILE=ArduinoCore-${PACKAGE}-${CORE_HASH}.tar.bz2 - PACKAGE_JSON=ArduinoCore-${PACKAGE}-${CORE_TAG}.json - ./extra/gen_package_index_json.sh ${CORE_TAG} ${ARTIFACT_FILE} > ${PACKAGE_JSON} + jq -cr '.[]' <<< ${ARTIFACTS} | while read -r artifact; do + ARTIFACT_FILE=ArduinoCore-${artifact}-${CORE_HASH}.tar.bz2 + PACKAGE_JSON=ArduinoCore-${artifact}-${CORE_TAG}.json + ./extra/gen_package_index_json.sh ${artifact} ${ARTIFACT_FILE} ${PACKAGE_JSON} + done - - name: Archive package index snippet + - name: Archive package index snippets uses: actions/upload-artifact@v4 with: name: ArduinoCore-zephyr-${{ env.CORE_TAG }}-jsons path: ArduinoCore-*-${{ env.CORE_TAG }}.json + + cleanup-artifacts: + runs-on: ubuntu-latest + needs: + - package-core + - publish-core + - publish-json + if: ${{ !cancelled() }} + steps: + - name: Clean up intermediate artifacts + uses: geekyeggo/delete-artifact@v5.1.0 + with: + name: ArduinoCore-archives-${{ needs.package-core.outputs.CORE_HASH }} + failOnError: false diff --git a/boards.txt b/boards.txt index 872ccc75..779ba90d 100644 --- a/boards.txt +++ b/boards.txt @@ -17,6 +17,7 @@ giga.build.zephyr_target=arduino_giga_r1//m7 giga.build.zephyr_args=--shield arduino_giga_display_shield giga.build.zephyr_hals=hal_stm32 hal_infineon giga.build.variant=arduino_giga_r1_stm32h747xx_m7 +giga.build.artifact=zephyr_main giga.build.mcu=cortex-m7 giga.build.fpu=-mfpu=fpv5-d16 giga.build.architecture=cortex-m7 @@ -78,6 +79,7 @@ nano33ble.menu.debug.true.build.zsk_args.debug=-debug nano33ble.build.zephyr_target=arduino_nano_33_ble//sense nano33ble.build.zephyr_args= nano33ble.build.zephyr_hals=hal_nordic +nano33ble.build.artifact=zephyr_main nano33ble.build.variant=arduino_nano_33_ble_nrf52840_sense nano33ble.build.mcu=cortex-m4 nano33ble.build.fpu=-mfpu=fpv4-sp-d16 @@ -126,7 +128,7 @@ nano33ble.debug.cortex-debug.custom.request=attach ############################################################################################################## -ek_ra8d1.name=EK_RA8D1 +ek_ra8d1.name=Renesas RA8D1 EK ek_ra8d1.build.core=arduino ek_ra8d1.build.crossprefix=arm-zephyr-eabi- ek_ra8d1.build.compiler_path={runtime.tools.arm-zephyr-eabi-0.16.8.path}/bin/ @@ -180,7 +182,7 @@ ek_ra8d1.bootloader.target=R7FA8D1AH ############################################################################################################## -frdm_mcxn947.name=MCXN947 +frdm_mcxn947.name=NXP FRDM MCXN947 frdm_mcxn947.build.core=arduino frdm_mcxn947.build.crossprefix=arm-zephyr-eabi- frdm_mcxn947.build.compiler_path={runtime.tools.arm-zephyr-eabi-0.16.8.path}/bin/ @@ -241,6 +243,7 @@ portentah7.menu.debug.true.build.zsk_args.debug=-debug portentah7.build.zephyr_target=arduino_portenta_h7@1.0.0//m7 portentah7.build.zephyr_args= portentah7.build.zephyr_hals=hal_stm32 hal_infineon +portentah7.build.artifact=zephyr_main portentah7.build.variant=arduino_portenta_h7_stm32h747xx_m7 portentah7.build.mcu=cortex-m7 portentah7.build.fpu=-mfpu=fpv5-d16 @@ -291,7 +294,7 @@ portentah7.debug.svd_file={runtime.platform.path}/svd/STM32H747_CM7.svd ############################################################################################################## -frdm_rw612.name=RW612 +frdm_rw612.name=NXP FRDM RW612 frdm_rw612.build.core=arduino frdm_rw612.build.crossprefix=arm-zephyr-eabi- frdm_rw612.build.compiler_path={runtime.tools.arm-zephyr-eabi-0.16.8.path}/bin/ @@ -351,6 +354,7 @@ niclasense.menu.debug.true.build.zsk_args.debug=-debug niclasense.build.zephyr_target=arduino_nicla_sense_me niclasense.build.zephyr_args= niclasense.build.zephyr_hals=hal_nordic +niclasense.build.artifact=zephyr_main niclasense.build.variant=arduino_nicla_sense_me_nrf52832 niclasense.build.mcu=cortex-m4 niclasense.build.fpu=-mfpu=fpv4-sp-d16 @@ -405,7 +409,7 @@ niclasense.debug.cortex-debug.custom.request=attach ########################################################################################## -portentac33.name=Portenta C33 +portentac33.name=Arduino Portenta C33 portentac33.build.core=arduino portentac33.build.crossprefix=arm-zephyr-eabi- portentac33.build.compiler_path={runtime.tools.arm-zephyr-eabi-0.16.8.path}/bin/ @@ -422,6 +426,7 @@ portentac33.menu.link_mode.static.upload.extension=bin-zsk.bin portentac33.build.zephyr_target=arduino_portenta_c33 portentac33.build.zephyr_args= portentac33.build.zephyr_hals=hal_renesas nanopb +portentac33.build.artifact=zephyr_main portentac33.build.variant=arduino_portenta_c33_r7fa6m5bh3cfc portentac33.build.mcu=cortex-m33 portentac33.build.fpu=-mfpu=fpv5-sp-d16 @@ -473,6 +478,7 @@ opta.menu.debug.true.build.zsk_args.debug=-debug opta.build.zephyr_target=arduino_opta//m7 opta.build.zephyr_args= opta.build.zephyr_hals=hal_stm32 hal_infineon +opta.build.artifact=zephyr_main opta.build.variant=arduino_opta_stm32h747xx_m7 opta.build.mcu=cortex-m7 opta.build.fpu=-mfpu=fpv5-d16 @@ -541,6 +547,8 @@ unoq.menu.flash_mode.ram.openocd_cfg=flash_sketch_ram.cfg unoq.build.zephyr_target=arduino_uno_q unoq.build.zephyr_args= unoq.build.variant=arduino_uno_q_stm32u585xx +unoq.build.artifact=zephyr_unoq +unoq.build.subarch=zephyr unoq.build.mcu=cortex-m33 unoq.build.fpu=-mfpu=fpv5-sp-d16 unoq.build.architecture=cortex-m33 diff --git a/extra/package_core.exc b/extra/artifacts/_common.exc similarity index 100% rename from extra/package_core.exc rename to extra/artifacts/_common.exc diff --git a/extra/package_core.inc b/extra/artifacts/_common.inc similarity index 87% rename from extra/package_core.inc rename to extra/artifacts/_common.inc index e084f321..754a1e6d 100644 --- a/extra/package_core.inc +++ b/extra/artifacts/_common.inc @@ -5,8 +5,7 @@ # are automatically added by package_core.sh depending on the information # in 'boards.txt', so they should not be included here. -boards.txt -# platform.txt is added by package_core.sh +# boards.txt and platform.txt are added by package_core.sh programmers.txt LICENSE README.md diff --git a/extra/artifacts/_common.json b/extra/artifacts/_common.json new file mode 100644 index 00000000..c1b6b08b --- /dev/null +++ b/extra/artifacts/_common.json @@ -0,0 +1,51 @@ +{ + "name": "", + "architecture": "", + "version": "", + "category": "", + "url": "", + "archiveFileName": "", + "checksum": "", + "size": "", + "help": { + "online": "https://www.arduino.cc/en/Reference/HomePage" + }, + "boards": [], + "toolsDependencies": [ + { + "packager": "zephyr", + "name": "arm-zephyr-eabi", + "version": "0.16.8" + }, + { + "packager": "arduino", + "name": "dfu-util", + "version": "0.11.0-arduino5" + }, + { + "packager": "arduino", + "name": "bossac", + "version": "1.9.1-arduino2" + }, + { + "packager": "arduino", + "name": "zephyr-sketch-tool", + "version": "0.1.0" + }, + { + "packager": "arduino", + "name": "sync-zephyr-artifacts", + "version": "0.1.0" + }, + { + "packager": "arduino", + "name": "adb", + "version": "32.0.0" + }, + { + "packager": "arduino", + "name": "remoteocd", + "version": "0.0.4-rc.4" + } + ] +} diff --git a/extra/artifacts/zephyr_contrib.json b/extra/artifacts/zephyr_contrib.json new file mode 100644 index 00000000..9fa9b8a5 --- /dev/null +++ b/extra/artifacts/zephyr_contrib.json @@ -0,0 +1,5 @@ +{ + "name": "Zephyr community boards (BETA)", + "architecture": "zephyr_contrib", + "category": "Contributed" +} diff --git a/extra/artifacts/zephyr_main.json b/extra/artifacts/zephyr_main.json new file mode 100644 index 00000000..b36cef34 --- /dev/null +++ b/extra/artifacts/zephyr_main.json @@ -0,0 +1,5 @@ +{ + "name": "Arduino Zephyr Boards (BETA)", + "architecture": "zephyr_main", + "category": "Arduino" +} diff --git a/extra/artifacts/zephyr_unoq.json b/extra/artifacts/zephyr_unoq.json new file mode 100644 index 00000000..96426b60 --- /dev/null +++ b/extra/artifacts/zephyr_unoq.json @@ -0,0 +1,5 @@ +{ + "name": "Arduino Uno Q board", + "architecture": "zephyr", + "category": "Arduino" +} diff --git a/extra/build_all.sh b/extra/build_all.sh index f132b83d..db3b8529 100755 --- a/extra/build_all.sh +++ b/extra/build_all.sh @@ -27,6 +27,7 @@ fi final_result=0 while read -r item; do board=$(jq -cr '.board' <<< "$item") + subarch=$(jq -cr '.subarch' <<< "$item") variant=$(jq -cr '.variant' <<< "$item") target=$(jq -cr '.target' <<< "$item") args=$(jq -cr '.args // ""' <<< "$item") @@ -36,7 +37,7 @@ while read -r item; do echo "${board} (${variant})" echo "${board} (${variant})" | sed -e 's/./=/g' else - echo "::group::=== ${board} (${variant}) ===" + echo "::group::=== ${subarch}:${board} (${variant}) ===" fi ./extra/build.sh "$target" $args @@ -49,10 +50,10 @@ while read -r item; do else echo "::endgroup::" if [ $result -eq 0 ] ; then - echo "- :white_check_mark: \`${variant}\`" >> "$GITHUB_STEP_SUMMARY" + echo "- :white_check_mark: \`${variant}\` (${subarch})" >> "$GITHUB_STEP_SUMMARY" else - echo "^^$(echo "=== ${board} (${variant}) ===" | sed -e 's/./^/g') FAILED with $result!" - echo "- :x: \`${variant}\`" >> "$GITHUB_STEP_SUMMARY" + echo "^^$(echo "=== ${subarch}:${board} (${variant}) ===" | sed -e 's/./^/g') FAILED with $result!" + echo "- :x: \`${variant}\` (${subarch})" >> "$GITHUB_STEP_SUMMARY" fi fi [ $result -ne 0 ] && ! $FORCE && exit $result diff --git a/extra/gen_package_index_json.sh b/extra/gen_package_index_json.sh index d86df9bf..218eeec6 100755 --- a/extra/gen_package_index_json.sh +++ b/extra/gen_package_index_json.sh @@ -1,22 +1,42 @@ #!/bin/bash -if [ $# -ne 2 ] ; then - echo "Usage: $0 " - exit 1 +set -e + +if [ $# -lt 2 ] || [ $# -gt 3 ] ; then + echo "Usage: $0 []" 1>&2 + exit 1 fi -PACKAGE=zephyr -CORE_TAG=$1 +ARTIFACT=$1 ARTIFACT_FILE=$2 +JSON_FILE=${3:-/dev/stdout} if [ ! -f "$ARTIFACT_FILE" ] ; then echo "Artifact file '$ARTIFACT_FILE' not found" 1>&2 exit 1 fi -JSON_TEMPLATE="extra/zephyr-core-template.json" -cat $JSON_TEMPLATE | sed \ - -e "s/__CORE_TAG__/$(extra/get_core_version.sh)/" \ - -e "s/__ARTIFACT_FILE__/$(basename $ARTIFACT_FILE)/" \ - -e "s/__ARTIFACT_HASH__/$(sha256sum $ARTIFACT_FILE | awk '{print $1}')/" \ - -e "s/__ARTIFACT_SIZE__/$(stat -c %s $ARTIFACT_FILE)/" | jq . +if [ "$ARTIFACT" == "zephyr" ] ; then + exit 0 # no JSON needed for the merged file +fi + +if ! [ -f "extra/artifacts/$ARTIFACT.json" ] ; then + echo "Artifact '$ARTIFACT' not found" 1>&2 + exit 1 +fi + +BOARD_NAMES=$(extra/get_board_details.sh | jq -c "map(select(.artifact == \"$ARTIFACT\")) | map({name}) | sort") + +jq -s '{ "packages": [ { "platforms": [ .[0]*.[1]*.[2] ] } ] }' \ + extra/artifacts/_common.json \ + extra/artifacts/${ARTIFACT}.json \ + - > "$JSON_FILE" << EOF +{ + "version": "$(extra/get_core_version.sh)", + "url": "https://downloads.arduino.cc/cores/zephyr/$(basename $ARTIFACT_FILE)", + "archiveFileName": "$(basename $ARTIFACT_FILE)", + "checksum": "SHA256:$(sha256sum $ARTIFACT_FILE | awk '{print $1}')", + "size": "$(stat -c %s $ARTIFACT_FILE)", + "boards": $BOARD_NAMES +} +EOF diff --git a/extra/get_board_details.sh b/extra/get_board_details.sh index a92b850d..b3506c3c 100755 --- a/extra/get_board_details.sh +++ b/extra/get_board_details.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -e + get_boards() { cat boards.txt | sed -e 's/#.*//' | grep -E '^.*\.build\.variant=' | sed -e 's/\.build\.variant=.*//' } @@ -16,6 +18,15 @@ for BOARD in $(get_boards); do TARGET=$(get_board_field $BOARD "build\\.zephyr_target") ARGS=$(get_board_field $BOARD "build\\.zephyr_args") HALS=$(get_board_field $BOARD "build\\.zephyr_hals") + ARTIFACT=$(get_board_field $BOARD "build\\.artifact") + ARTIFACT=${ARTIFACT:-zephyr_contrib} + + ARTIFACT_JSON=extra/artifacts/$ARTIFACT.json + if ! [ -f "$ARTIFACT_JSON" ] ; then + echo "error: missing artifact description file $ARTIFACT_JSON" 1>&2 + exit 1 + fi + SUBARCH=$(jq -r '.architecture' < $ARTIFACT_JSON) if [ -z "$TARGET" ] ; then echo "error: missing '$BOARD.build.zephyr_target'" 1>&2 @@ -34,6 +45,8 @@ for BOARD in $(get_boards); do "target": "$TARGET", "args": "$ARGS", "hals": "$HALS", + "artifact": "$ARTIFACT", + "subarch": "$SUBARCH" } EOF done | jq -crs . diff --git a/extra/package_core.sh b/extra/package_core.sh index 9af06163..bc95aec1 100755 --- a/extra/package_core.sh +++ b/extra/package_core.sh @@ -3,7 +3,7 @@ set -e if [ -z "$1" ]; then - echo "Usage: $0 VERSION [OUTPUT_FILE]" + echo "Usage: $0 ARTIFACT VERSION [OUTPUT_FILE]" exit 1 fi @@ -12,13 +12,41 @@ if [ ! -f platform.txt ]; then exit 2 fi -PACKAGE=zephyr -VERSION=$1 -OUTPUT_FILE=${2:-distrib/${PACKAGE}-${VERSION}.tar.bz2} +ARTIFACT=$1 +VERSION=$2 +OUTPUT_FILE=${3:-distrib/${ARTIFACT}-${VERSION}.tar.bz2} + +# we use variants for include because we filter on file paths +# and boards for exclude because we want to remove matching lines in boards.txt +BOARD_DETAILS=$(extra/get_board_details.sh) +if [ $ARTIFACT == "zephyr" ] ; then + INCLUDED_VARIANTS=$(echo ${BOARD_DETAILS} | jq -cr ".[].variant") + EXCLUDED_BOARDS="" +elif [ ! -f extra/artifacts/${ARTIFACT}.json ]; then + echo "Unknown artifact '$ARTIFACT'." + exit 3 +else + ARTIFACT_NAME="$(grep '"name"' extra/artifacts/${ARTIFACT}.json | head -n 1 | cut -d '"' -f 4) (${VERSION})" + INCLUDED_VARIANTS=$(echo ${BOARD_DETAILS} | jq -cr "map(select(.artifact == \"$ARTIFACT\")) | .[].variant") + EXCLUDED_BOARDS=$(echo ${BOARD_DETAILS} | jq -cr "map(select(.artifact != \"$ARTIFACT\")) | .[].board") +fi + +[ -n $GITHUB_WORKSPACE ] && echo "::group::Packaging ${ARTIFACT_NAME:-all variants} ($(basename $OUTPUT_FILE))" + +# create a temporary boards.txt file with the correct list of boards +TEMP_BOARDS=$(mktemp -p . | sed 's/\.\///') +cat boards.txt >> $TEMP_BOARDS +for board in $EXCLUDED_BOARDS ; do + # remove (even commented) lines for excluded boards + sed -i "/^\(\s*#\s*\)\?${board}\./d" $TEMP_BOARDS +done +# remove multiple empty lines +sed -i '/^$/N;/^\n$/D' $TEMP_BOARDS # create a temporary platform.txt file with the correct version TEMP_PLATFORM=$(mktemp -p . | sed 's/\.\///') cat platform.txt > ${TEMP_PLATFORM} +[ -z "$ARTIFACT_NAME" ] || sed -ie "s/^name=.*/name=${ARTIFACT_NAME}/" ${TEMP_PLATFORM} sed -ie "s/^version=.*/version=$(extra/get_core_version.sh)/" ${TEMP_PLATFORM} declutter_file() { @@ -28,21 +56,26 @@ declutter_file() { # create the list of files and directories to include TEMP_INC=$(mktemp -p . | sed 's/\.\///') +echo ${TEMP_BOARDS} >> ${TEMP_INC} echo ${TEMP_PLATFORM} >> ${TEMP_INC} -declutter_file extra/package_core.inc >> ${TEMP_INC} -extra/get_board_details.sh | jq -cr '.[]' | while read -r item; do - variant=$(jq -cr '.variant' <<< "$item") - echo "::info::`${variant}`" +declutter_file extra/artifacts/_common.inc >> ${TEMP_INC} +declutter_file extra/artifacts/$ARTIFACT.inc >> ${TEMP_INC} +for variant in $INCLUDED_VARIANTS ; do + echo "::info::\`${variant}\`" echo "variants/${variant}/" >> ${TEMP_INC} ls firmwares/zephyr-${variant}.* >> ${TEMP_INC} done # create the list of files and directories to exclude TEMP_EXC=$(mktemp -p . | sed 's/\.\///') -declutter_file extra/package.exc >> ${TEMP_EXC} +declutter_file extra/artifacts/_common.exc >> ${TEMP_EXC} +declutter_file extra/artifacts/$ARTIFACT.exc >> ${TEMP_EXC} mkdir -p $(dirname ${OUTPUT_FILE}) tar -cjhf ${OUTPUT_FILE} -X ${TEMP_EXC} -T ${TEMP_INC} \ + --transform "s,${TEMP_BOARDS},boards.txt," \ --transform "s,${TEMP_PLATFORM},platform.txt," \ --transform "s,^,ArduinoCore-zephyr/," -rm -f ${TEMP_INC} ${TEMP_EXC} ${TEMP_PLATFORM} +rm -f ${TEMP_INC} ${TEMP_EXC} ${TEMP_BOARDS} ${TEMP_PLATFORM} + +[ -n $GITHUB_WORKSPACE ] && echo "::endgroup::" diff --git a/extra/zephyr-core-template.json b/extra/zephyr-core-template.json deleted file mode 100644 index 5c945f28..00000000 --- a/extra/zephyr-core-template.json +++ /dev/null @@ -1,87 +0,0 @@ -{ - "packages": [ - { - "platforms": [ - { - "name": "Arduino Zephyr Boards (BETA)", - "architecture": "zephyr", - "version": "__CORE_TAG__", - "category": "Arduino", - "url": "https://downloads.arduino.cc/cores/zephyr/__ARTIFACT_FILE__", - "archiveFileName": "__ARTIFACT_FILE__", - "checksum": "SHA-256:__ARTIFACT_HASH__", - "size": "__ARTIFACT_SIZE__", - "help": { - "online": "https://www.arduino.cc/en/Reference/HomePage" - }, - "boards": [ - { - "name": "Arduino Giga" - }, - { - "name": "Arduino Nano 33 BLE" - }, - { - "name": "Arduino Opta" - }, - { - "name": "Arduino Portenta H7" - }, - { - "name": "Arduino Portenta C33" - }, - { - "name": "Renesas EK_RA8D1" - }, - { - "name": "NXP FRDM MCXN947" - }, - { - "name": "NXP FRDM RW612" - }, - { - "name": "Arduino UNO Q" - } - ], - "toolsDependencies": [ - { - "packager": "zephyr", - "name": "arm-zephyr-eabi", - "version": "0.16.8" - }, - { - "packager": "arduino", - "name": "dfu-util", - "version": "0.11.0-arduino5" - }, - { - "packager": "arduino", - "name": "bossac", - "version": "1.9.1-arduino2" - }, - { - "packager": "arduino", - "name": "zephyr-sketch-tool", - "version": "0.1.0" - }, - { - "packager": "arduino", - "name": "sync-zephyr-artifacts", - "version": "0.1.0" - }, - { - "packager": "arduino", - "name": "adb", - "version": "32.0.0" - }, - { - "packager": "arduino", - "name": "remoteocd", - "version": "0.0.3-rc.2" - } - ] - } - ] - } - ] -} From efe6e9cd11147cf81d494899e25ef7c20454a607 Mon Sep 17 00:00:00 2001 From: Luca Burelli Date: Tue, 14 Oct 2025 18:15:26 +0200 Subject: [PATCH 5/5] ci/package_core: generate JSONs on PRs too Make sure the JSONs are generated on PRs too, so that they can be inspected before merging. --- .github/workflows/package_core.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/package_core.yml b/.github/workflows/package_core.yml index ebf73fd3..899b6256 100644 --- a/.github/workflows/package_core.yml +++ b/.github/workflows/package_core.yml @@ -266,7 +266,7 @@ jobs: publish-json: name: Publish jsons runs-on: ubuntu-latest - if: ${{ github.event_name == 'push' && github.repository == 'arduino/ArduinoCore-zephyr' }} + if: ${{ github.repository == 'arduino/ArduinoCore-zephyr' }} needs: - package-core - test-core