diff --git a/.github/workflows/package_core.yml b/.github/workflows/package_core.yml index 887a67d81..899b62561 100644 --- a/.github/workflows/package_core.yml +++ b/.github/workflows/package_core.yml @@ -7,18 +7,21 @@ on: jobs: package-core: - name: Build and package core + name: Build and package cores runs-on: ubuntu-latest env: CCACHE_IGNOREOPTIONS: -specs=* outputs: CORE_TAG: ${{ env.CORE_TAG }} - CORE_ARTIFACT: ${{ env.CORE_ARTIFACT }} - BOARD_VARIANTS: ${{ env.BOARD_VARIANTS }} + CORE_HASH: ${{ env.CORE_HASH }} + ALL_BOARD_DATA: ${{ env.ALL_BOARD_DATA }} + ARTIFACTS: ${{ env.ARTIFACTS }} + SUB_ARCHES: ${{ env.SUB_ARCHES }} steps: - name: Install OS dependencies working-directory: /opt run: | + sudo apt-get remove --purge man-db -y # skips the mandb triggers sudo apt-get update sudo apt-get install -y --no-install-recommends git cmake wget python3-pip ninja-build ccache @@ -32,9 +35,16 @@ jobs: - name: Initialize Zephyr environment run: | yes | ./extra/bootstrap.sh -o=--filter=tree:0 - echo "CORE_TAG=$(git describe --tags --exact-match || git describe --always)" >> "$GITHUB_ENV" - echo "CORE_ARTIFACT=ArduinoCore-zephyr-$(git describe --always)" >> "$GITHUB_ENV" - echo "BOARD_VARIANTS=$(extra/get_board_details.sh | jq -cr 'sort_by(.variant)')" >> "$GITHUB_ENV" + echo "CORE_HASH=$(git describe --always)" >> "$GITHUB_ENV" + echo "ALL_BOARD_DATA=$(extra/get_board_details.sh | jq -c 'sort_by(.variant)')" >> "$GITHUB_ENV" + echo "## Building \`$(extra/get_core_version.sh)\`" >> "$GITHUB_STEP_SUMMARY" + + - name: Map output packages + # needs the above env vars to be usable + run: | + echo "CORE_TAG=$(git describe --tags --exact-match 2>/dev/null || echo $CORE_HASH)" >> "$GITHUB_ENV" + echo "ARTIFACTS=$(jq -c '["zephyr"] + (map(.artifact) | unique)' <<< ${ALL_BOARD_DATA})" >> "$GITHUB_ENV" + echo "SUB_ARCHES=$(jq -c 'map(.subarch) | unique' <<< ${ALL_BOARD_DATA})" >> "$GITHUB_ENV" - name: ccache uses: hendrikmuhs/ccache-action@v1.2 @@ -46,36 +56,63 @@ jobs: run: | ./extra/build_all.sh -f - - name: Package core + - name: Package cores run: | - ./extra/package_core.sh ${{ env.CORE_TAG }} distrib/${{ env.CORE_ARTIFACT }}.tar.bz2 + jq -cr '.[]' <<< ${ARTIFACTS} | while read -r artifact; do + ARTIFACT_NAME=ArduinoCore-${artifact}-${CORE_HASH} + ./extra/package_core.sh ${artifact} ${CORE_TAG} distrib/${ARTIFACT_NAME}.tar.bz2 + done - - name: Archive core + - name: Archive cores uses: actions/upload-artifact@v4 + with: + name: ArduinoCore-archives-${{ env.CORE_HASH }} + path: distrib/*.tar.bz2 + + split-core: + name: Split off ${{ matrix.artifact }} + runs-on: ubuntu-latest + needs: package-core + env: + ALL_CORES_ARTIFACT: ArduinoCore-archives-${{ needs.package-core.outputs.CORE_HASH }} + CORE_ARTIFACT: ArduinoCore-${{ matrix.artifact }}-${{ needs.package-core.outputs.CORE_HASH }} + strategy: + matrix: + artifact: ${{ fromJSON( needs.package-core.outputs.ARTIFACTS ) }} + steps: + - uses: actions/download-artifact@v4 + with: + name: ${{ env.ALL_CORES_ARTIFACT }} + + - uses: actions/upload-artifact@v4 with: name: ${{ env.CORE_ARTIFACT }} - path: distrib/${{ env.CORE_ARTIFACT }}.tar.bz2 + path: ${{ env.CORE_ARTIFACT }}.tar.bz2 test-core: - name: Test on ${{ matrix.board }} + name: Test ${{ matrix.subarch }}:${{ matrix.board }} runs-on: ubuntu-latest - needs: package-core + needs: + - package-core + - split-core strategy: matrix: include: - ${{ fromJSON( needs.package-core.outputs.BOARD_VARIANTS ) }} + ${{ fromJSON( needs.package-core.outputs.ALL_BOARD_DATA ) }} fail-fast: false env: - FQBN: arduino:zephyr:${{ matrix.board }} - REPORT_FILE: arduino-zephyr-${{ matrix.board }}.json + PLAT: arduino:${{ matrix.subarch }} + FQBN: arduino:${{ matrix.subarch }}:${{ matrix.board }} + CORE_ARTIFACT: ArduinoCore-${{ matrix.artifact }}-${{ needs.package-core.outputs.CORE_HASH }} steps: - uses: actions/download-artifact@v4 with: - name: ${{ needs.package-core.outputs.CORE_ARTIFACT }} + name: ${{ env.CORE_ARTIFACT }} - name: Set up core run: | - tar xf ${{ needs.package-core.outputs.CORE_ARTIFACT }}.tar.bz2 + tar xf ${CORE_ARTIFACT}.tar.bz2 # will create ArduinoCore-zephyr/ + echo "REPORT_FILE=$(echo ${FQBN} | tr ':' '-').json" >> $GITHUB_ENV - name: Create Blink sketch run: | @@ -90,7 +127,7 @@ jobs: # Use Board Manager to install the latest release of Arduino Zephyr Boards to get the toolchain - name: "arduino:zephyr" source-url: "https://downloads.arduino.cc/packages/package_zephyr_index.json" - - name: "arduino:zephyr" + - name: ${{ env.PLAT }} source-path: "ArduinoCore-zephyr" sketch-paths: Blink cli-compile-flags: | @@ -114,7 +151,7 @@ jobs: repo: context.repo.repo, run_id: context.runId }); - const job_name = `Test on ${{ matrix.board }}` + const job_name = `Test ${{ matrix.subarch }}:${{ matrix.board }}` return workflow_run.jobs.find((job) => job.name === job_name).id; - name: Prepare log @@ -137,7 +174,7 @@ jobs: - test-core if: ${{ !cancelled() && needs.package-core.result == 'success' }} env: - BOARD_VARIANTS: ${{ needs.package-core.outputs.BOARD_VARIANTS }} + ALL_BOARD_DATA: ${{ needs.package-core.outputs.ALL_BOARD_DATA }} steps: - uses: actions/download-artifact@v4 with: @@ -146,37 +183,41 @@ jobs: merge-multiple: true - run: | - echo "### Core test results:" >> "$GITHUB_STEP_SUMMARY" - jq -c '.[]' <<< "$BOARD_VARIANTS" | while read -r BOARD_VARIANT; do - BOARD=$(echo $BOARD_VARIANT | jq -cr '.board') - VARIANT=$(echo $BOARD_VARIANT | jq -cr '.variant') - FQBN="arduino:zephyr:$BOARD" - REPORT_FILE="arduino-zephyr-$BOARD.json" - if [ ! -f $REPORT_FILE ]; then - echo "* :x: $BOARD (`$VARIANT`) - No report found?" >> "$GITHUB_STEP_SUMMARY" - else - REPORT=$(jq -cr '.boards[0].sketches[0]' $REPORT_FILE) - JOB_ID=$(echo $REPORT | jq -cr '.job_id') - JOB_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/job/${JOB_ID}#step:5:2" - if ! $(echo $REPORT | jq -cr '.compilation_success') ; then - echo "* :x: [$BOARD]($JOB_URL) (\`$VARIANT\`) - Build failed" >> "$GITHUB_STEP_SUMMARY" + ARTIFACTS=$(jq -cr 'map(.artifact) | unique | .[]' <<< ${ALL_BOARD_DATA}) # this avoids the 'zephyr' artifact + for artifact in $ARTIFACTS ; do + echo "### \`$artifact\` test results:" >> "$GITHUB_STEP_SUMMARY" + jq -c "map(select(.artifact == \"$artifact\")) | .[]" <<< ${ALL_BOARD_DATA} | while read -r BOARD_DATA; do + BOARD=$(echo $BOARD_DATA | jq -cr '.board') + VARIANT=$(echo $BOARD_DATA | jq -cr '.variant') + SUBARCH=$(echo $BOARD_DATA | jq -cr '.subarch') + FQBN="arduino:$SUBARCH:$BOARD" + REPORT_FILE="$(echo $FQBN | tr ':' '-').json" + if [ ! -f $REPORT_FILE ]; then + echo "* :x: $BOARD (\`$VARIANT\`) - No report found?" >> "$GITHUB_STEP_SUMMARY" else - WARNINGS=$(echo $REPORT | jq -cr '.warnings.current.absolute // 0') - if [ $WARNINGS -eq 0 ]; then - echo "* :white_check_mark: $BOARD (\`$VARIANT\`) - Build successful" >> "$GITHUB_STEP_SUMMARY" + REPORT=$(jq -cr '.boards[0].sketches[0]' $REPORT_FILE) + JOB_ID=$(echo $REPORT | jq -cr '.job_id') + JOB_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/job/${JOB_ID}#step:5:2" + if ! $(echo $REPORT | jq -cr '.compilation_success') ; then + echo "* :x: [$BOARD]($JOB_URL) (\`$VARIANT\`) - Build failed" >> "$GITHUB_STEP_SUMMARY" else - echo >> "$GITHUB_STEP_SUMMARY" - echo "
   :warning: $BOARD ($VARIANT) - $WARNINGS warnings:" >> "$GITHUB_STEP_SUMMARY" - echo >> "$GITHUB_STEP_SUMMARY" - echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY" - echo $REPORT | jq -cr '.warnings_log[]' >> "$GITHUB_STEP_SUMMARY" - echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY" - echo >> "$GITHUB_STEP_SUMMARY" - echo "
" >> "$GITHUB_STEP_SUMMARY" - echo >> "$GITHUB_STEP_SUMMARY" + WARNINGS=$(echo $REPORT | jq -cr '.warnings.current.absolute // 0') + if [ $WARNINGS -eq 0 ]; then + echo "* :white_check_mark: $BOARD (\`$VARIANT\`) - Build successful" >> "$GITHUB_STEP_SUMMARY" + else + echo >> "$GITHUB_STEP_SUMMARY" + echo "
   :warning: $BOARD ($VARIANT) - $WARNINGS warnings:" >> "$GITHUB_STEP_SUMMARY" + echo >> "$GITHUB_STEP_SUMMARY" + echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY" + echo $REPORT | jq -cr '.warnings_log[]' >> "$GITHUB_STEP_SUMMARY" + echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY" + echo >> "$GITHUB_STEP_SUMMARY" + echo "
" >> "$GITHUB_STEP_SUMMARY" + echo >> "$GITHUB_STEP_SUMMARY" + fi fi fi - fi + done done - name: Clean up intermediate artifacts @@ -207,12 +248,11 @@ jobs: id-token: write contents: read env: - CORE_ARTIFACT: ${{ needs.package-core.outputs.CORE_ARTIFACT }} - ARTIFACT_FILE: ${{ needs.package-core.outputs.CORE_ARTIFACT }}.tar.bz2 + ALL_CORES_ARTIFACT: ArduinoCore-archives-${{ needs.package-core.outputs.CORE_HASH }} steps: - uses: actions/download-artifact@v4 with: - name: ${{ env.CORE_ARTIFACT }} + name: ${{ env.ALL_CORES_ARTIFACT }} - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 @@ -221,40 +261,55 @@ jobs: aws-region: ${{ secrets.AWS_REGION }} - name: Upload artifact - run: aws s3 cp ${{ env.ARTIFACT_FILE }} s3://${{ secrets.S3_BUCKET }}/ + run: aws s3 cp ArduinoCore-*.tar.bz2 s3://${{ secrets.S3_BUCKET }}/ publish-json: - name: Publish json + name: Publish jsons runs-on: ubuntu-latest - if: ${{ github.event_name == 'push' && github.repository == 'arduino/ArduinoCore-zephyr' }} + if: ${{ github.repository == 'arduino/ArduinoCore-zephyr' }} needs: - package-core - test-core - - publish-core env: - CORE_ARTIFACT: ${{ needs.package-core.outputs.CORE_ARTIFACT }} - ARTIFACT_FILE: ${{ needs.package-core.outputs.CORE_ARTIFACT }}.tar.bz2 + ALL_CORES_ARTIFACT: ArduinoCore-archives-${{ needs.package-core.outputs.CORE_HASH }} CORE_TAG: ${{ needs.package-core.outputs.CORE_TAG }} - PACKAGE_INDEX_JSON: zephyr-core-${{ needs.package-core.outputs.CORE_TAG }}.json + CORE_HASH: ${{ needs.package-core.outputs.CORE_HASH }} + ARTIFACTS: ${{ needs.package-core.outputs.ARTIFACTS }} steps: - uses: actions/checkout@v4 with: fetch-depth: 0 persist-credentials: false - sparse-checkout: | - extra/gen_package_index_json.sh - extra/zephyr-core-template.json + fetch-tags: true - uses: actions/download-artifact@v4 with: - name: ${{ env.CORE_ARTIFACT }} + name: ${{ env.ALL_CORES_ARTIFACT }} - # uses: ARTIFACT_FILE CORE_TAG PACKAGE_INDEX_JSON - - name: Prepare package index snippet - run: ./extra/gen_package_index_json.sh + - name: Prepare package index snippets + run: | + jq -cr '.[]' <<< ${ARTIFACTS} | while read -r artifact; do + ARTIFACT_FILE=ArduinoCore-${artifact}-${CORE_HASH}.tar.bz2 + PACKAGE_JSON=ArduinoCore-${artifact}-${CORE_TAG}.json + ./extra/gen_package_index_json.sh ${artifact} ${ARTIFACT_FILE} ${PACKAGE_JSON} + done - - name: Archive package index snippet + - name: Archive package index snippets uses: actions/upload-artifact@v4 with: - name: ${{ env.PACKAGE_INDEX_JSON }} - path: ${{ env.PACKAGE_INDEX_JSON }} + name: ArduinoCore-zephyr-${{ env.CORE_TAG }}-jsons + path: ArduinoCore-*-${{ env.CORE_TAG }}.json + + cleanup-artifacts: + runs-on: ubuntu-latest + needs: + - package-core + - publish-core + - publish-json + if: ${{ !cancelled() }} + steps: + - name: Clean up intermediate artifacts + uses: geekyeggo/delete-artifact@v5.1.0 + with: + name: ArduinoCore-archives-${{ needs.package-core.outputs.CORE_HASH }} + failOnError: false diff --git a/boards.txt b/boards.txt index 872ccc75b..779ba90d9 100644 --- a/boards.txt +++ b/boards.txt @@ -17,6 +17,7 @@ giga.build.zephyr_target=arduino_giga_r1//m7 giga.build.zephyr_args=--shield arduino_giga_display_shield giga.build.zephyr_hals=hal_stm32 hal_infineon giga.build.variant=arduino_giga_r1_stm32h747xx_m7 +giga.build.artifact=zephyr_main giga.build.mcu=cortex-m7 giga.build.fpu=-mfpu=fpv5-d16 giga.build.architecture=cortex-m7 @@ -78,6 +79,7 @@ nano33ble.menu.debug.true.build.zsk_args.debug=-debug nano33ble.build.zephyr_target=arduino_nano_33_ble//sense nano33ble.build.zephyr_args= nano33ble.build.zephyr_hals=hal_nordic +nano33ble.build.artifact=zephyr_main nano33ble.build.variant=arduino_nano_33_ble_nrf52840_sense nano33ble.build.mcu=cortex-m4 nano33ble.build.fpu=-mfpu=fpv4-sp-d16 @@ -126,7 +128,7 @@ nano33ble.debug.cortex-debug.custom.request=attach ############################################################################################################## -ek_ra8d1.name=EK_RA8D1 +ek_ra8d1.name=Renesas RA8D1 EK ek_ra8d1.build.core=arduino ek_ra8d1.build.crossprefix=arm-zephyr-eabi- ek_ra8d1.build.compiler_path={runtime.tools.arm-zephyr-eabi-0.16.8.path}/bin/ @@ -180,7 +182,7 @@ ek_ra8d1.bootloader.target=R7FA8D1AH ############################################################################################################## -frdm_mcxn947.name=MCXN947 +frdm_mcxn947.name=NXP FRDM MCXN947 frdm_mcxn947.build.core=arduino frdm_mcxn947.build.crossprefix=arm-zephyr-eabi- frdm_mcxn947.build.compiler_path={runtime.tools.arm-zephyr-eabi-0.16.8.path}/bin/ @@ -241,6 +243,7 @@ portentah7.menu.debug.true.build.zsk_args.debug=-debug portentah7.build.zephyr_target=arduino_portenta_h7@1.0.0//m7 portentah7.build.zephyr_args= portentah7.build.zephyr_hals=hal_stm32 hal_infineon +portentah7.build.artifact=zephyr_main portentah7.build.variant=arduino_portenta_h7_stm32h747xx_m7 portentah7.build.mcu=cortex-m7 portentah7.build.fpu=-mfpu=fpv5-d16 @@ -291,7 +294,7 @@ portentah7.debug.svd_file={runtime.platform.path}/svd/STM32H747_CM7.svd ############################################################################################################## -frdm_rw612.name=RW612 +frdm_rw612.name=NXP FRDM RW612 frdm_rw612.build.core=arduino frdm_rw612.build.crossprefix=arm-zephyr-eabi- frdm_rw612.build.compiler_path={runtime.tools.arm-zephyr-eabi-0.16.8.path}/bin/ @@ -351,6 +354,7 @@ niclasense.menu.debug.true.build.zsk_args.debug=-debug niclasense.build.zephyr_target=arduino_nicla_sense_me niclasense.build.zephyr_args= niclasense.build.zephyr_hals=hal_nordic +niclasense.build.artifact=zephyr_main niclasense.build.variant=arduino_nicla_sense_me_nrf52832 niclasense.build.mcu=cortex-m4 niclasense.build.fpu=-mfpu=fpv4-sp-d16 @@ -405,7 +409,7 @@ niclasense.debug.cortex-debug.custom.request=attach ########################################################################################## -portentac33.name=Portenta C33 +portentac33.name=Arduino Portenta C33 portentac33.build.core=arduino portentac33.build.crossprefix=arm-zephyr-eabi- portentac33.build.compiler_path={runtime.tools.arm-zephyr-eabi-0.16.8.path}/bin/ @@ -422,6 +426,7 @@ portentac33.menu.link_mode.static.upload.extension=bin-zsk.bin portentac33.build.zephyr_target=arduino_portenta_c33 portentac33.build.zephyr_args= portentac33.build.zephyr_hals=hal_renesas nanopb +portentac33.build.artifact=zephyr_main portentac33.build.variant=arduino_portenta_c33_r7fa6m5bh3cfc portentac33.build.mcu=cortex-m33 portentac33.build.fpu=-mfpu=fpv5-sp-d16 @@ -473,6 +478,7 @@ opta.menu.debug.true.build.zsk_args.debug=-debug opta.build.zephyr_target=arduino_opta//m7 opta.build.zephyr_args= opta.build.zephyr_hals=hal_stm32 hal_infineon +opta.build.artifact=zephyr_main opta.build.variant=arduino_opta_stm32h747xx_m7 opta.build.mcu=cortex-m7 opta.build.fpu=-mfpu=fpv5-d16 @@ -541,6 +547,8 @@ unoq.menu.flash_mode.ram.openocd_cfg=flash_sketch_ram.cfg unoq.build.zephyr_target=arduino_uno_q unoq.build.zephyr_args= unoq.build.variant=arduino_uno_q_stm32u585xx +unoq.build.artifact=zephyr_unoq +unoq.build.subarch=zephyr unoq.build.mcu=cortex-m33 unoq.build.fpu=-mfpu=fpv5-sp-d16 unoq.build.architecture=cortex-m33 diff --git a/extra/package_core.exc b/extra/artifacts/_common.exc similarity index 100% rename from extra/package_core.exc rename to extra/artifacts/_common.exc diff --git a/extra/package_core.inc b/extra/artifacts/_common.inc similarity index 87% rename from extra/package_core.inc rename to extra/artifacts/_common.inc index e084f3213..754a1e6d3 100644 --- a/extra/package_core.inc +++ b/extra/artifacts/_common.inc @@ -5,8 +5,7 @@ # are automatically added by package_core.sh depending on the information # in 'boards.txt', so they should not be included here. -boards.txt -# platform.txt is added by package_core.sh +# boards.txt and platform.txt are added by package_core.sh programmers.txt LICENSE README.md diff --git a/extra/artifacts/_common.json b/extra/artifacts/_common.json new file mode 100644 index 000000000..c1b6b08b7 --- /dev/null +++ b/extra/artifacts/_common.json @@ -0,0 +1,51 @@ +{ + "name": "", + "architecture": "", + "version": "", + "category": "", + "url": "", + "archiveFileName": "", + "checksum": "", + "size": "", + "help": { + "online": "https://www.arduino.cc/en/Reference/HomePage" + }, + "boards": [], + "toolsDependencies": [ + { + "packager": "zephyr", + "name": "arm-zephyr-eabi", + "version": "0.16.8" + }, + { + "packager": "arduino", + "name": "dfu-util", + "version": "0.11.0-arduino5" + }, + { + "packager": "arduino", + "name": "bossac", + "version": "1.9.1-arduino2" + }, + { + "packager": "arduino", + "name": "zephyr-sketch-tool", + "version": "0.1.0" + }, + { + "packager": "arduino", + "name": "sync-zephyr-artifacts", + "version": "0.1.0" + }, + { + "packager": "arduino", + "name": "adb", + "version": "32.0.0" + }, + { + "packager": "arduino", + "name": "remoteocd", + "version": "0.0.4-rc.4" + } + ] +} diff --git a/extra/artifacts/zephyr_contrib.json b/extra/artifacts/zephyr_contrib.json new file mode 100644 index 000000000..9fa9b8a5c --- /dev/null +++ b/extra/artifacts/zephyr_contrib.json @@ -0,0 +1,5 @@ +{ + "name": "Zephyr community boards (BETA)", + "architecture": "zephyr_contrib", + "category": "Contributed" +} diff --git a/extra/artifacts/zephyr_main.json b/extra/artifacts/zephyr_main.json new file mode 100644 index 000000000..b36cef34b --- /dev/null +++ b/extra/artifacts/zephyr_main.json @@ -0,0 +1,5 @@ +{ + "name": "Arduino Zephyr Boards (BETA)", + "architecture": "zephyr_main", + "category": "Arduino" +} diff --git a/extra/artifacts/zephyr_unoq.json b/extra/artifacts/zephyr_unoq.json new file mode 100644 index 000000000..96426b604 --- /dev/null +++ b/extra/artifacts/zephyr_unoq.json @@ -0,0 +1,5 @@ +{ + "name": "Arduino Uno Q board", + "architecture": "zephyr", + "category": "Arduino" +} diff --git a/extra/bootstrap.sh b/extra/bootstrap.sh index 99c7f3cc7..bf63e65d3 100755 --- a/extra/bootstrap.sh +++ b/extra/bootstrap.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -e + if [ ! -f platform.txt ]; then echo Launch this script from the root core folder as ./extra/bootstrap.sh exit 2 diff --git a/extra/build_all.sh b/extra/build_all.sh index f132b83dc..db3b8529c 100755 --- a/extra/build_all.sh +++ b/extra/build_all.sh @@ -27,6 +27,7 @@ fi final_result=0 while read -r item; do board=$(jq -cr '.board' <<< "$item") + subarch=$(jq -cr '.subarch' <<< "$item") variant=$(jq -cr '.variant' <<< "$item") target=$(jq -cr '.target' <<< "$item") args=$(jq -cr '.args // ""' <<< "$item") @@ -36,7 +37,7 @@ while read -r item; do echo "${board} (${variant})" echo "${board} (${variant})" | sed -e 's/./=/g' else - echo "::group::=== ${board} (${variant}) ===" + echo "::group::=== ${subarch}:${board} (${variant}) ===" fi ./extra/build.sh "$target" $args @@ -49,10 +50,10 @@ while read -r item; do else echo "::endgroup::" if [ $result -eq 0 ] ; then - echo "- :white_check_mark: \`${variant}\`" >> "$GITHUB_STEP_SUMMARY" + echo "- :white_check_mark: \`${variant}\` (${subarch})" >> "$GITHUB_STEP_SUMMARY" else - echo "^^$(echo "=== ${board} (${variant}) ===" | sed -e 's/./^/g') FAILED with $result!" - echo "- :x: \`${variant}\`" >> "$GITHUB_STEP_SUMMARY" + echo "^^$(echo "=== ${subarch}:${board} (${variant}) ===" | sed -e 's/./^/g') FAILED with $result!" + echo "- :x: \`${variant}\` (${subarch})" >> "$GITHUB_STEP_SUMMARY" fi fi [ $result -ne 0 ] && ! $FORCE && exit $result diff --git a/extra/gen_package_index_json.sh b/extra/gen_package_index_json.sh index 6cb0eb10d..218eeec6d 100755 --- a/extra/gen_package_index_json.sh +++ b/extra/gen_package_index_json.sh @@ -1,13 +1,42 @@ #!/bin/bash -if [ -z "$CORE_TAG" ]; then - echo "This script can be used in Github CI only." - exit 1 +set -e + +if [ $# -lt 2 ] || [ $# -gt 3 ] ; then + echo "Usage: $0 []" 1>&2 + exit 1 fi -cat extra/zephyr-core-template.json | sed \ - -e "s/__CORE_TAG__/$CORE_TAG/" \ - -e "s/__ARTIFACT_FILE__/$ARTIFACT_FILE/" \ - -e "s/__ARTIFACT_HASH__/$(sha256sum $ARTIFACT_FILE | awk '{print $1}')/" \ - -e "s/__ARTIFACT_SIZE__/$(stat -c %s $ARTIFACT_FILE)/" \ - > $PACKAGE_INDEX_JSON +ARTIFACT=$1 +ARTIFACT_FILE=$2 +JSON_FILE=${3:-/dev/stdout} + +if [ ! -f "$ARTIFACT_FILE" ] ; then + echo "Artifact file '$ARTIFACT_FILE' not found" 1>&2 + exit 1 +fi + +if [ "$ARTIFACT" == "zephyr" ] ; then + exit 0 # no JSON needed for the merged file +fi + +if ! [ -f "extra/artifacts/$ARTIFACT.json" ] ; then + echo "Artifact '$ARTIFACT' not found" 1>&2 + exit 1 +fi + +BOARD_NAMES=$(extra/get_board_details.sh | jq -c "map(select(.artifact == \"$ARTIFACT\")) | map({name}) | sort") + +jq -s '{ "packages": [ { "platforms": [ .[0]*.[1]*.[2] ] } ] }' \ + extra/artifacts/_common.json \ + extra/artifacts/${ARTIFACT}.json \ + - > "$JSON_FILE" << EOF +{ + "version": "$(extra/get_core_version.sh)", + "url": "https://downloads.arduino.cc/cores/zephyr/$(basename $ARTIFACT_FILE)", + "archiveFileName": "$(basename $ARTIFACT_FILE)", + "checksum": "SHA256:$(sha256sum $ARTIFACT_FILE | awk '{print $1}')", + "size": "$(stat -c %s $ARTIFACT_FILE)", + "boards": $BOARD_NAMES +} +EOF diff --git a/extra/get_board_details.sh b/extra/get_board_details.sh index 035b101d3..b3506c3cc 100755 --- a/extra/get_board_details.sh +++ b/extra/get_board_details.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -e + get_boards() { cat boards.txt | sed -e 's/#.*//' | grep -E '^.*\.build\.variant=' | sed -e 's/\.build\.variant=.*//' } @@ -7,14 +9,24 @@ get_boards() { get_board_field() { board=$1 field=$2 - cat boards.txt | sed -e 's/#.*//' | grep -E "^$board\\.build\\.$field=" | cut -d '=' -f2- | sed -e 's/"/\"/g' + cat boards.txt | sed -e 's/#.*//' | grep -E "^$board\\.$field=" | cut -d '=' -f2- | sed -e 's/"/\"/g' } for BOARD in $(get_boards); do - VARIANT=$(get_board_field $BOARD "variant") - TARGET=$(get_board_field $BOARD "zephyr_target") - ARGS=$(get_board_field $BOARD "zephyr_args") - HALS=$(get_board_field $BOARD "zephyr_hals") + NAME=$(get_board_field $BOARD "name") + VARIANT=$(get_board_field $BOARD "build\\.variant") + TARGET=$(get_board_field $BOARD "build\\.zephyr_target") + ARGS=$(get_board_field $BOARD "build\\.zephyr_args") + HALS=$(get_board_field $BOARD "build\\.zephyr_hals") + ARTIFACT=$(get_board_field $BOARD "build\\.artifact") + ARTIFACT=${ARTIFACT:-zephyr_contrib} + + ARTIFACT_JSON=extra/artifacts/$ARTIFACT.json + if ! [ -f "$ARTIFACT_JSON" ] ; then + echo "error: missing artifact description file $ARTIFACT_JSON" 1>&2 + exit 1 + fi + SUBARCH=$(jq -r '.architecture' < $ARTIFACT_JSON) if [ -z "$TARGET" ] ; then echo "error: missing '$BOARD.build.zephyr_target'" 1>&2 @@ -25,5 +37,16 @@ for BOARD in $(get_boards); do exit 1 fi - echo "{ \"board\": \"$BOARD\", \"variant\": \"$VARIANT\", \"target\": \"$TARGET\", \"args\": \"$ARGS\", \"hals\": \"$HALS\" }" + cat << EOF + { + "name": "$NAME", + "board": "$BOARD", + "variant": "$VARIANT", + "target": "$TARGET", + "args": "$ARGS", + "hals": "$HALS", + "artifact": "$ARTIFACT", + "subarch": "$SUBARCH" + } +EOF done | jq -crs . diff --git a/extra/get_core_version.sh b/extra/get_core_version.sh new file mode 100755 index 000000000..5cfdf6b7c --- /dev/null +++ b/extra/get_core_version.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# This script generates a SemVer-compatible version number based on Git tags. +# +# If the current commit is tagged, it returns that version. If not, it +# generates a version string based on the next patch number and the current +# commit hash. +# +# If the tag is a simple "..", git describe will output: +# +# ..--g +# +# The regexp splits the patch number, awk converts the tokens to: +# +# ..-0.dev+ +# +# If the tag refers to a pre-release, like "..-", +# sed will output a total of 4 arguments and the format of the final awk output +# will be: +# +# ..--0.dev+ +# +# These are among the lowest possible SemVer versions greater than the last +# tagged version. +# +# If there are no tags at all (for example when run in a fork etc), it defaults +# to "9.9.9-+". + +VERSION=$(git describe --tags --exact-match 2>/dev/null) +if [ -z "$VERSION" ]; then + VERSION=$(git describe --tags --dirty 2>/dev/null | + sed 's/\.\([[:digit:]]\+\)\(-.*\)*-[[:digit:]]\+-g/ \1 \2 /' | + awk '{ if (NF==3) { print $1 "." ($2+1) "-0.dev+" $3 } else { print $1 "." $2 $3 "-0.dev+" $4 }}') + if [ -z "$VERSION" ]; then + VERSION="9.9.9-$(date '+%Y%m%d-%H%M%S')+$(git describe --always --dirty)" + fi +fi +echo $VERSION diff --git a/extra/package_core.sh b/extra/package_core.sh index 3b0b46ae6..bc95aec1b 100755 --- a/extra/package_core.sh +++ b/extra/package_core.sh @@ -3,36 +3,79 @@ set -e if [ -z "$1" ]; then - echo "Usage: $0 VERSION [OUTPUT_FILE]" + echo "Usage: $0 ARTIFACT VERSION [OUTPUT_FILE]" exit 1 fi if [ ! -f platform.txt ]; then - echo "Launch this script from the root core folder as ./extra/package_core.sh VERSION" + echo "Launch this script from the root core folder." exit 2 fi -PACKAGE=ArduinoCore-zephyr -VERSION=$1 -OUTPUT_FILE=${2:-distrib/${PACKAGE}-${VERSION}.tar.bz2} +ARTIFACT=$1 +VERSION=$2 +OUTPUT_FILE=${3:-distrib/${ARTIFACT}-${VERSION}.tar.bz2} + +# we use variants for include because we filter on file paths +# and boards for exclude because we want to remove matching lines in boards.txt +BOARD_DETAILS=$(extra/get_board_details.sh) +if [ $ARTIFACT == "zephyr" ] ; then + INCLUDED_VARIANTS=$(echo ${BOARD_DETAILS} | jq -cr ".[].variant") + EXCLUDED_BOARDS="" +elif [ ! -f extra/artifacts/${ARTIFACT}.json ]; then + echo "Unknown artifact '$ARTIFACT'." + exit 3 +else + ARTIFACT_NAME="$(grep '"name"' extra/artifacts/${ARTIFACT}.json | head -n 1 | cut -d '"' -f 4) (${VERSION})" + INCLUDED_VARIANTS=$(echo ${BOARD_DETAILS} | jq -cr "map(select(.artifact == \"$ARTIFACT\")) | .[].variant") + EXCLUDED_BOARDS=$(echo ${BOARD_DETAILS} | jq -cr "map(select(.artifact != \"$ARTIFACT\")) | .[].board") +fi + +[ -n $GITHUB_WORKSPACE ] && echo "::group::Packaging ${ARTIFACT_NAME:-all variants} ($(basename $OUTPUT_FILE))" + +# create a temporary boards.txt file with the correct list of boards +TEMP_BOARDS=$(mktemp -p . | sed 's/\.\///') +cat boards.txt >> $TEMP_BOARDS +for board in $EXCLUDED_BOARDS ; do + # remove (even commented) lines for excluded boards + sed -i "/^\(\s*#\s*\)\?${board}\./d" $TEMP_BOARDS +done +# remove multiple empty lines +sed -i '/^$/N;/^\n$/D' $TEMP_BOARDS # create a temporary platform.txt file with the correct version TEMP_PLATFORM=$(mktemp -p . | sed 's/\.\///') -sed -e "s/^version=.*/version=${VERSION}/" platform.txt > ${TEMP_PLATFORM} +cat platform.txt > ${TEMP_PLATFORM} +[ -z "$ARTIFACT_NAME" ] || sed -ie "s/^name=.*/name=${ARTIFACT_NAME}/" ${TEMP_PLATFORM} +sed -ie "s/^version=.*/version=$(extra/get_core_version.sh)/" ${TEMP_PLATFORM} -TEMP_LIST=$(mktemp) -echo ${TEMP_PLATFORM} > ${TEMP_LIST} +declutter_file() { + [ -f "$1" ] || return 0 + cat "$1" | sed -e 's/\s*#.*//' | grep -v '^\s*$' +} -# import a basic list of files and directories -cat extra/package_core.inc | sed -e 's/\s*#.*//' | grep -v '^\s*$' >> ${TEMP_LIST} - -# add the board-specific files -extra/get_board_details.sh | jq -cr '.[]' | while read -r item; do - variant=$(jq -cr '.variant' <<< "$item") - echo "variants/${variant}/" >> ${TEMP_LIST} - ls firmwares/zephyr-${variant}.* >> ${TEMP_LIST} +# create the list of files and directories to include +TEMP_INC=$(mktemp -p . | sed 's/\.\///') +echo ${TEMP_BOARDS} >> ${TEMP_INC} +echo ${TEMP_PLATFORM} >> ${TEMP_INC} +declutter_file extra/artifacts/_common.inc >> ${TEMP_INC} +declutter_file extra/artifacts/$ARTIFACT.inc >> ${TEMP_INC} +for variant in $INCLUDED_VARIANTS ; do + echo "::info::\`${variant}\`" + echo "variants/${variant}/" >> ${TEMP_INC} + ls firmwares/zephyr-${variant}.* >> ${TEMP_INC} done -cat ${TEMP_LIST} + +# create the list of files and directories to exclude +TEMP_EXC=$(mktemp -p . | sed 's/\.\///') +declutter_file extra/artifacts/_common.exc >> ${TEMP_EXC} +declutter_file extra/artifacts/$ARTIFACT.exc >> ${TEMP_EXC} + mkdir -p $(dirname ${OUTPUT_FILE}) -tar -cjhf ${OUTPUT_FILE} -X extra/package_core.exc -T ${TEMP_LIST} --transform "s,${TEMP_PLATFORM},platform.txt," --transform "s,^,${PACKAGE}/," -rm -f ${TEMP_LIST} ${TEMP_PLATFORM} +tar -cjhf ${OUTPUT_FILE} -X ${TEMP_EXC} -T ${TEMP_INC} \ + --transform "s,${TEMP_BOARDS},boards.txt," \ + --transform "s,${TEMP_PLATFORM},platform.txt," \ + --transform "s,^,ArduinoCore-zephyr/," +rm -f ${TEMP_INC} ${TEMP_EXC} ${TEMP_BOARDS} ${TEMP_PLATFORM} + +[ -n $GITHUB_WORKSPACE ] && echo "::endgroup::" diff --git a/extra/zephyr-core-template.json b/extra/zephyr-core-template.json deleted file mode 100644 index 5c945f28f..000000000 --- a/extra/zephyr-core-template.json +++ /dev/null @@ -1,87 +0,0 @@ -{ - "packages": [ - { - "platforms": [ - { - "name": "Arduino Zephyr Boards (BETA)", - "architecture": "zephyr", - "version": "__CORE_TAG__", - "category": "Arduino", - "url": "https://downloads.arduino.cc/cores/zephyr/__ARTIFACT_FILE__", - "archiveFileName": "__ARTIFACT_FILE__", - "checksum": "SHA-256:__ARTIFACT_HASH__", - "size": "__ARTIFACT_SIZE__", - "help": { - "online": "https://www.arduino.cc/en/Reference/HomePage" - }, - "boards": [ - { - "name": "Arduino Giga" - }, - { - "name": "Arduino Nano 33 BLE" - }, - { - "name": "Arduino Opta" - }, - { - "name": "Arduino Portenta H7" - }, - { - "name": "Arduino Portenta C33" - }, - { - "name": "Renesas EK_RA8D1" - }, - { - "name": "NXP FRDM MCXN947" - }, - { - "name": "NXP FRDM RW612" - }, - { - "name": "Arduino UNO Q" - } - ], - "toolsDependencies": [ - { - "packager": "zephyr", - "name": "arm-zephyr-eabi", - "version": "0.16.8" - }, - { - "packager": "arduino", - "name": "dfu-util", - "version": "0.11.0-arduino5" - }, - { - "packager": "arduino", - "name": "bossac", - "version": "1.9.1-arduino2" - }, - { - "packager": "arduino", - "name": "zephyr-sketch-tool", - "version": "0.1.0" - }, - { - "packager": "arduino", - "name": "sync-zephyr-artifacts", - "version": "0.1.0" - }, - { - "packager": "arduino", - "name": "adb", - "version": "32.0.0" - }, - { - "packager": "arduino", - "name": "remoteocd", - "version": "0.0.3-rc.2" - } - ] - } - ] - } - ] -}