ci/package_core: multi-artifact support

Add a new "board.build.artifact" option to boards.txt to specify which
artifact a board belongs to. If not specified, the board will
automatically be added to "zephyr_contrib".

The artifact names collected from boards.txt (plus the auto "zephyr"
package, which includes all boards) are used to build and package each
artifact in the GitHub Actions workflow; in particular, the FQBN of each
board will depend on its artifact's "architecture" field.
The test action is also updated to test each artifact on the appropriate
boards.

Individual package JSON files are in the extra/artifacts directory,
along with common include/exclude files for packaging.

Signed-off-by: Luca Burelli <[email protected]>

Author: pillo79

.github/workflows/package_core.yml

@@ -7,14 +7,16 @@ on:
 jobs:
 
   package-core:
-    name: Build and package core
+    name: Build and package cores
     runs-on: ubuntu-latest
     env:
       CCACHE_IGNOREOPTIONS: -specs=*
     outputs:
       CORE_TAG: ${{ env.CORE_TAG }}
       CORE_HASH: ${{ env.CORE_HASH }}
       ALL_BOARD_DATA: ${{ env.ALL_BOARD_DATA }}
+      ARTIFACTS: ${{ env.ARTIFACTS }}
+      SUB_ARCHES: ${{ env.SUB_ARCHES }}
     steps:
       - name: Install OS dependencies
         working-directory: /opt
@@ -54,32 +56,54 @@ jobs:
         run: |
           ./extra/build_all.sh -f
 
-      - name: Package core
+      - name: Package cores
         run: |
-          PACKAGE=zephyr
-          CORE_ARTIFACT=ArduinoCore-$PACKAGE-${CORE_HASH}
-          ./extra/package_core.sh ${CORE_TAG} distrib/${CORE_ARTIFACT}.tar.bz2
+          jq -cr '.[]' <<< ${ARTIFACTS} | while read -r artifact; do
+            ARTIFACT_NAME=ArduinoCore-${artifact}-${CORE_HASH}
+            ./extra/package_core.sh ${artifact} ${CORE_TAG} distrib/${ARTIFACT_NAME}.tar.bz2
+          done
 
-      - name: Archive core
+      - name: Archive cores
         uses: actions/upload-artifact@v4
         with:
-          name: ArduinoCore-zephyr-${{ env.CORE_HASH }}
+          name: ArduinoCore-archives-${{ env.CORE_HASH }}
           path: distrib/*.tar.bz2
 
+  split-core:
+    name: Split off ${{ matrix.artifact }}
+    runs-on: ubuntu-latest
+    needs: package-core
+    env:
+      ALL_CORES_ARTIFACT: ArduinoCore-archives-${{ needs.package-core.outputs.CORE_HASH }}
+      CORE_ARTIFACT: ArduinoCore-${{ matrix.artifact }}-${{ needs.package-core.outputs.CORE_HASH }}
+    strategy:
+      matrix:
+        artifact: ${{ fromJSON( needs.package-core.outputs.ARTIFACTS ) }}
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: ${{ env.ALL_CORES_ARTIFACT }}
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.CORE_ARTIFACT }}
+          path: ${{ env.CORE_ARTIFACT }}.tar.bz2
+
   test-core:
-    name: Test on ${{ matrix.board }}
+    name: Test ${{ matrix.subarch }}:${{ matrix.board }}
     runs-on: ubuntu-latest
     needs:
       - package-core
+      - split-core
     strategy:
       matrix:
         include:
           ${{ fromJSON( needs.package-core.outputs.ALL_BOARD_DATA ) }}
       fail-fast: false
     env:
-      PLAT: arduino:zephyr
-      FQBN: arduino:zephyr:${{ matrix.board }}
-      CORE_ARTIFACT: ArduinoCore-zephyr-${{ needs.package-core.outputs.CORE_HASH }}
+      PLAT: arduino:${{ matrix.subarch }}
+      FQBN: arduino:${{ matrix.subarch }}:${{ matrix.board }}
+      CORE_ARTIFACT: ArduinoCore-${{ matrix.artifact }}-${{ needs.package-core.outputs.CORE_HASH }}
     steps:
       - uses: actions/download-artifact@v4
         with:
@@ -127,7 +151,7 @@ jobs:
               repo: context.repo.repo,
               run_id: context.runId
             });
-            const job_name = `Test on ${{ matrix.board }}`
+            const job_name = `Test ${{ matrix.subarch }}:${{ matrix.board }}`
             return workflow_run.jobs.find((job) => job.name === job_name).id;
 
       - name: Prepare log
@@ -159,37 +183,41 @@ jobs:
           merge-multiple: true
 
       - run: |
-          echo "### Core test results:" >> "$GITHUB_STEP_SUMMARY"
-          jq -c '.[]' <<< "$ALL_BOARD_DATA" | while read -r BOARD_DATA; do
-            BOARD=$(echo $BOARD_DATA | jq -cr '.board')
-            VARIANT=$(echo $BOARD_DATA | jq -cr '.variant')
-            FQBN="arduino:zephyr:$BOARD"
-            REPORT_FILE="$(echo $FQBN | tr ':' '-').json"
-            if [ ! -f $REPORT_FILE ]; then
-              echo "* :x: $BOARD (`$VARIANT`) - No report found?" >> "$GITHUB_STEP_SUMMARY"
-            else
-              REPORT=$(jq -cr '.boards[0].sketches[0]' $REPORT_FILE)
-              JOB_ID=$(echo $REPORT | jq -cr '.job_id')
-              JOB_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/job/${JOB_ID}#step:5:2"
-              if ! $(echo $REPORT | jq -cr '.compilation_success') ; then
-                echo "* :x: [$BOARD]($JOB_URL) (\`$VARIANT\`) - Build failed" >> "$GITHUB_STEP_SUMMARY"
+          ARTIFACTS=$(jq -cr 'map(.artifact) | unique | .[]' <<< ${ALL_BOARD_DATA}) # this avoids the 'zephyr' artifact
+          for artifact in $ARTIFACTS ; do
+            echo "### \`$artifact\` test results:" >> "$GITHUB_STEP_SUMMARY"
+            jq -c "map(select(.artifact == \"$artifact\")) | .[]" <<< ${ALL_BOARD_DATA} | while read -r BOARD_DATA; do
+              BOARD=$(echo $BOARD_DATA | jq -cr '.board')
+              VARIANT=$(echo $BOARD_DATA | jq -cr '.variant')
+              SUBARCH=$(echo $BOARD_DATA | jq -cr '.subarch')
+              FQBN="arduino:$SUBARCH:$BOARD"
+              REPORT_FILE="$(echo $FQBN | tr ':' '-').json"
+              if [ ! -f $REPORT_FILE ]; then
+                echo "* :x: $BOARD (\`$VARIANT\`) - No report found?" >> "$GITHUB_STEP_SUMMARY"
               else
-                WARNINGS=$(echo $REPORT | jq -cr '.warnings.current.absolute // 0')
-                if [ $WARNINGS -eq 0 ]; then
-                  echo "* :white_check_mark: $BOARD (\`$VARIANT\`) - Build successful" >> "$GITHUB_STEP_SUMMARY"
+                REPORT=$(jq -cr '.boards[0].sketches[0]' $REPORT_FILE)
+                JOB_ID=$(echo $REPORT | jq -cr '.job_id')
+                JOB_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/job/${JOB_ID}#step:5:2"
+                if ! $(echo $REPORT | jq -cr '.compilation_success') ; then
+                  echo "* :x: [$BOARD]($JOB_URL) (\`$VARIANT\`) - Build failed" >> "$GITHUB_STEP_SUMMARY"
                 else
-                  echo >> "$GITHUB_STEP_SUMMARY"
-                  echo "<details><summary>&nbsp;&nbsp; :warning: <a href=\"$JOB_URL\">$BOARD</a> (<tt>$VARIANT</tt>) - $WARNINGS warnings:</summary>" >> "$GITHUB_STEP_SUMMARY"
-                  echo >> "$GITHUB_STEP_SUMMARY"
-                  echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY"
-                  echo $REPORT | jq -cr '.warnings_log[]' >> "$GITHUB_STEP_SUMMARY"
-                  echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY"
-                  echo >> "$GITHUB_STEP_SUMMARY"
-                  echo "</details>" >> "$GITHUB_STEP_SUMMARY"
-                  echo >> "$GITHUB_STEP_SUMMARY"
+                  WARNINGS=$(echo $REPORT | jq -cr '.warnings.current.absolute // 0')
+                  if [ $WARNINGS -eq 0 ]; then
+                    echo "* :white_check_mark: $BOARD (\`$VARIANT\`) - Build successful" >> "$GITHUB_STEP_SUMMARY"
+                  else
+                    echo >> "$GITHUB_STEP_SUMMARY"
+                    echo "<details><summary>&nbsp;&nbsp; :warning: <a href=\"$JOB_URL\">$BOARD</a> (<tt>$VARIANT</tt>) - $WARNINGS warnings:</summary>" >> "$GITHUB_STEP_SUMMARY"
+                    echo >> "$GITHUB_STEP_SUMMARY"
+                    echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY"
+                    echo $REPORT | jq -cr '.warnings_log[]' >> "$GITHUB_STEP_SUMMARY"
+                    echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY"
+                    echo >> "$GITHUB_STEP_SUMMARY"
+                    echo "</details>" >> "$GITHUB_STEP_SUMMARY"
+                    echo >> "$GITHUB_STEP_SUMMARY"
+                  fi
                 fi
               fi
-            fi
+            done
           done
 
       - name: Clean up intermediate artifacts
@@ -220,11 +248,11 @@ jobs:
       id-token: write
       contents: read
     env:
-      CORE_ARTIFACT: ArduinoCore-zephyr-${{ needs.package-core.outputs.CORE_HASH }}
+      ALL_CORES_ARTIFACT: ArduinoCore-archives-${{ needs.package-core.outputs.CORE_HASH }}
     steps:
       - uses: actions/download-artifact@v4
         with:
-          name: ${{ env.CORE_ARTIFACT }}
+          name: ${{ env.ALL_CORES_ARTIFACT }}
 
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v4
@@ -236,37 +264,52 @@ jobs:
         run: aws s3 cp ArduinoCore-*.tar.bz2 s3://${{ secrets.S3_BUCKET }}/
 
   publish-json:
-    name: Publish json
+    name: Publish jsons
     runs-on: ubuntu-latest
     if: ${{ github.event_name == 'push' && github.repository == 'arduino/ArduinoCore-zephyr' }}
     needs:
       - package-core
       - test-core
     env:
-      CORE_ARTIFACT: ArduinoCore-zephyr-${{ needs.package-core.outputs.CORE_HASH }}
+      ALL_CORES_ARTIFACT: ArduinoCore-archives-${{ needs.package-core.outputs.CORE_HASH }}
       CORE_TAG: ${{ needs.package-core.outputs.CORE_TAG }}
+      CORE_HASH: ${{ needs.package-core.outputs.CORE_HASH }}
+      ARTIFACTS: ${{ needs.package-core.outputs.ARTIFACTS }}
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
           persist-credentials: false
-          sparse-checkout: |
-            extra/gen_package_index_json.sh
-            extra/zephyr-core-template.json
+          fetch-tags: true
 
       - uses: actions/download-artifact@v4
         with:
-          name: ${{ env.CORE_ARTIFACT }}
+          name: ${{ env.ALL_CORES_ARTIFACT }}
 
-      - name: Prepare package index snippet
+      - name: Prepare package index snippets
         run: |
-          PACKAGE=zephyr
-          ARTIFACT_FILE=ArduinoCore-${PACKAGE}-${CORE_HASH}.tar.bz2
-          PACKAGE_JSON=ArduinoCore-${PACKAGE}-${CORE_TAG}.json
-          ./extra/gen_package_index_json.sh ${CORE_TAG} ${ARTIFACT_FILE} > ${PACKAGE_JSON}
+          jq -cr '.[]' <<< ${ARTIFACTS} | while read -r artifact; do
+            ARTIFACT_FILE=ArduinoCore-${artifact}-${CORE_HASH}.tar.bz2
+            PACKAGE_JSON=ArduinoCore-${artifact}-${CORE_TAG}.json
+            ./extra/gen_package_index_json.sh ${artifact} ${ARTIFACT_FILE} ${PACKAGE_JSON}
+          done
 
-      - name: Archive package index snippet
+      - name: Archive package index snippets
         uses: actions/upload-artifact@v4
         with:
           name: ArduinoCore-zephyr-${{ env.CORE_TAG }}-jsons
           path: ArduinoCore-*-${{ env.CORE_TAG }}.json
+
+  cleanup-artifacts:
+    runs-on: ubuntu-latest
+    needs:
+      - package-core
+      - publish-core
+      - publish-json
+    if: ${{ !cancelled() }}
+    steps:
+      - name: Clean up intermediate artifacts
+        uses: geekyeggo/[email protected]
+        with:
+          name: ArduinoCore-archives-${{ needs.package-core.outputs.CORE_HASH }}
+          failOnError: false

boards.txt

@@ -17,6 +17,7 @@ giga.build.zephyr_target=arduino_giga_r1//m7
 giga.build.zephyr_args=--shield arduino_giga_display_shield
 giga.build.zephyr_hals=hal_stm32 hal_infineon
 giga.build.variant=arduino_giga_r1_stm32h747xx_m7
+giga.build.artifact=zephyr_main
 giga.build.mcu=cortex-m7
 giga.build.fpu=-mfpu=fpv5-d16
 giga.build.architecture=cortex-m7
@@ -78,6 +79,7 @@ nano33ble.menu.debug.true.build.zsk_args.debug=-debug
 nano33ble.build.zephyr_target=arduino_nano_33_ble//sense
 nano33ble.build.zephyr_args=
 nano33ble.build.zephyr_hals=hal_nordic
+nano33ble.build.artifact=zephyr_main
 nano33ble.build.variant=arduino_nano_33_ble_nrf52840_sense
 nano33ble.build.mcu=cortex-m4
 nano33ble.build.fpu=-mfpu=fpv4-sp-d16
@@ -126,7 +128,7 @@ nano33ble.debug.cortex-debug.custom.request=attach
 
 ##############################################################################################################
 
-ek_ra8d1.name=EK_RA8D1
+ek_ra8d1.name=Renesas RA8D1 EK
 ek_ra8d1.build.core=arduino
 ek_ra8d1.build.crossprefix=arm-zephyr-eabi-
 ek_ra8d1.build.compiler_path={runtime.tools.arm-zephyr-eabi-0.16.8.path}/bin/
@@ -180,7 +182,7 @@ ek_ra8d1.bootloader.target=R7FA8D1AH
 
 ##############################################################################################################
 
-frdm_mcxn947.name=MCXN947
+frdm_mcxn947.name=NXP FRDM MCXN947
 frdm_mcxn947.build.core=arduino
 frdm_mcxn947.build.crossprefix=arm-zephyr-eabi-
 frdm_mcxn947.build.compiler_path={runtime.tools.arm-zephyr-eabi-0.16.8.path}/bin/
@@ -241,6 +243,7 @@ portentah7.menu.debug.true.build.zsk_args.debug=-debug
 [email protected]//m7
 portentah7.build.zephyr_args=
 portentah7.build.zephyr_hals=hal_stm32 hal_infineon
+portentah7.build.artifact=zephyr_main
 portentah7.build.variant=arduino_portenta_h7_stm32h747xx_m7
 portentah7.build.mcu=cortex-m7
 portentah7.build.fpu=-mfpu=fpv5-d16
@@ -291,7 +294,7 @@ portentah7.debug.svd_file={runtime.platform.path}/svd/STM32H747_CM7.svd
 
 ##############################################################################################################
 
-frdm_rw612.name=RW612
+frdm_rw612.name=NXP FRDM RW612
 frdm_rw612.build.core=arduino
 frdm_rw612.build.crossprefix=arm-zephyr-eabi-
 frdm_rw612.build.compiler_path={runtime.tools.arm-zephyr-eabi-0.16.8.path}/bin/
@@ -351,6 +354,7 @@ niclasense.menu.debug.true.build.zsk_args.debug=-debug
 niclasense.build.zephyr_target=arduino_nicla_sense_me
 niclasense.build.zephyr_args=
 niclasense.build.zephyr_hals=hal_nordic
+niclasense.build.artifact=zephyr_main
 niclasense.build.variant=arduino_nicla_sense_me_nrf52832
 niclasense.build.mcu=cortex-m4
 niclasense.build.fpu=-mfpu=fpv4-sp-d16
@@ -405,7 +409,7 @@ niclasense.debug.cortex-debug.custom.request=attach
 
 ##########################################################################################
 
-portentac33.name=Portenta C33
+portentac33.name=Arduino Portenta C33
 portentac33.build.core=arduino
 portentac33.build.crossprefix=arm-zephyr-eabi-
 portentac33.build.compiler_path={runtime.tools.arm-zephyr-eabi-0.16.8.path}/bin/
@@ -422,6 +426,7 @@ portentac33.menu.link_mode.static.upload.extension=bin-zsk.bin
 portentac33.build.zephyr_target=arduino_portenta_c33
 portentac33.build.zephyr_args=
 portentac33.build.zephyr_hals=hal_renesas nanopb
+portentac33.build.artifact=zephyr_main
 portentac33.build.variant=arduino_portenta_c33_r7fa6m5bh3cfc
 portentac33.build.mcu=cortex-m33
 portentac33.build.fpu=-mfpu=fpv5-sp-d16
@@ -473,6 +478,7 @@ opta.menu.debug.true.build.zsk_args.debug=-debug
 opta.build.zephyr_target=arduino_opta//m7
 opta.build.zephyr_args=
 opta.build.zephyr_hals=hal_stm32 hal_infineon
+opta.build.artifact=zephyr_main
 opta.build.variant=arduino_opta_stm32h747xx_m7
 opta.build.mcu=cortex-m7
 opta.build.fpu=-mfpu=fpv5-d16
@@ -541,6 +547,8 @@ unoq.menu.flash_mode.ram.openocd_cfg=flash_sketch_ram.cfg
 unoq.build.zephyr_target=arduino_uno_q
 unoq.build.zephyr_args=
 unoq.build.variant=arduino_uno_q_stm32u585xx
+unoq.build.artifact=zephyr_unoq
+unoq.build.subarch=zephyr
 unoq.build.mcu=cortex-m33
 unoq.build.fpu=-mfpu=fpv5-sp-d16
 unoq.build.architecture=cortex-m33

extra/package_core.exc renamed to extra/artifacts/_common.exc

@@ -5,8 +5,7 @@
 #       are automatically added by package_core.sh depending on the information
 #       in 'boards.txt', so they should not be included here.
 
-boards.txt
-# platform.txt is added by package_core.sh
+# boards.txt and platform.txt are added by package_core.sh
 programmers.txt
 LICENSE
 README.md

extra/package_core.inc renamed to extra/artifacts/_common.inc

@@ -0,0 +1,51 @@
+{
+  "name": "",
+  "architecture": "",
+  "version": "",
+  "category": "",
+  "url": "",
+  "archiveFileName": "",
+  "checksum": "",
+  "size": "",
+  "help": {
+    "online": "https://www.arduino.cc/en/Reference/HomePage"
+  },
+  "boards": [],
+  "toolsDependencies": [
+    {
+      "packager": "zephyr",
+      "name": "arm-zephyr-eabi",
+      "version": "0.16.8"
+    },
+    {
+      "packager": "arduino",
+      "name": "dfu-util",
+      "version": "0.11.0-arduino5"
+    },
+    {
+      "packager": "arduino",
+      "name": "bossac",
+      "version": "1.9.1-arduino2"
+    },
+    {
+      "packager": "arduino",
+      "name": "zephyr-sketch-tool",
+      "version": "0.1.0"
+    },
+    {
+      "packager": "arduino",
+      "name": "sync-zephyr-artifacts",
+      "version": "0.1.0"
+    },
+    {
+      "packager": "arduino",
+      "name": "adb",
+      "version": "32.0.0"
+    },
+    {
+      "packager": "arduino",
+      "name": "remoteocd",
+      "version": "0.0.4-rc.4"
+    }
+  ]
+}