ci/package_core: refactor to prepare multi-package

pillo79 · pillo79 · commit 90751a1c5a7b · 2025-10-15T10:55:25.000+02:00
* Generate CORE_ARTIFACT every step so that multi-package is possible
  via matrix iterations in the next commit.

* Convert gen_package_index_json.sh to take parameters instead of
  environment variables.

* Export the name of the package from get_board_details.sh.

* Rework package_core.sh so that it is easily extendable to multiple
  packages.

Signed-off-by: Luca Burelli &lt;l.burelli@arduino.cc&gt;
diff --git a/.github/workflows/package_core.yml b/.github/workflows/package_core.yml
@@ -13,12 +13,13 @@ jobs:
       CCACHE_IGNOREOPTIONS: -specs=*
     outputs:
       CORE_TAG: ${{ env.CORE_TAG }}
-      CORE_ARTIFACT: ${{ env.CORE_ARTIFACT }}
+      CORE_HASH: ${{ env.CORE_HASH }}
       ALL_BOARD_DATA: ${{ env.ALL_BOARD_DATA }}
     steps:
       - name: Install OS dependencies
         working-directory: /opt
         run: |
+          sudo apt-get remove --purge man-db -y # skips the mandb triggers
           sudo apt-get update
           sudo apt-get install -y --no-install-recommends git cmake wget python3-pip ninja-build ccache
 
@@ -32,9 +33,15 @@ jobs:
       - name: Initialize Zephyr environment
         run: |
           yes | ./extra/bootstrap.sh -o=--filter=tree:0
-          echo "CORE_TAG=$(git describe --tags --exact-match || git describe --always)" >> "$GITHUB_ENV"
-          echo "CORE_ARTIFACT=ArduinoCore-zephyr-$(git describe --always)" >> "$GITHUB_ENV"
-          echo "ALL_BOARD_DATA=$(extra/get_board_details.sh | jq -cr 'sort_by(.variant)')" >> "$GITHUB_ENV"
+          echo "CORE_HASH=$(git describe --always)" >> "$GITHUB_ENV"
+          echo "ALL_BOARD_DATA=$(extra/get_board_details.sh | jq -c 'sort_by(.variant)')" >> "$GITHUB_ENV"
+
+      - name: Map output packages
+        # needs the above env vars to be usable
+        run: |
+          echo "CORE_TAG=$(git describe --tags --exact-match 2>/dev/null || echo $CORE_HASH)" >> "$GITHUB_ENV"
+          echo "ARTIFACTS=$(jq -c '["zephyr"] + (map(.artifact) | unique)' <<< ${ALL_BOARD_DATA})" >> "$GITHUB_ENV"
+          echo "SUB_ARCHES=$(jq -c 'map(.subarch) | unique' <<< ${ALL_BOARD_DATA})" >> "$GITHUB_ENV"
 
       - name: ccache
         uses: hendrikmuhs/ccache-action@v1.2
@@ -48,18 +55,21 @@ jobs:
 
       - name: Package core
         run: |
+          PACKAGE=zephyr
+          CORE_ARTIFACT=ArduinoCore-$PACKAGE-${CORE_HASH}
           ./extra/package_core.sh ${CORE_TAG} distrib/${CORE_ARTIFACT}.tar.bz2
 
       - name: Archive core
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ env.CORE_ARTIFACT }}
-          path: distrib/${{ env.CORE_ARTIFACT }}.tar.bz2
+          name: ArduinoCore-zephyr-${{ env.CORE_HASH }}
+          path: distrib/*.tar.bz2
 
   test-core:
     name: Test on ${{ matrix.board }}
     runs-on: ubuntu-latest
-    needs: package-core
+    needs:
+      - package-core
     strategy:
       matrix:
         include:
@@ -68,15 +78,15 @@ jobs:
     env:
       PLAT: arduino:zephyr
       FQBN: arduino:zephyr:${{ matrix.board }}
-      CORE_ARTIFACT: ${{ needs.package-core.outputs.CORE_ARTIFACT }}
+      CORE_ARTIFACT: ArduinoCore-zephyr-${{ needs.package-core.outputs.CORE_HASH }}
     steps:
       - uses: actions/download-artifact@v4
         with:
           name: ${{ env.CORE_ARTIFACT }}
 
       - name: Set up core
         run: |
-          tar xf ${CORE_ARTIFACT}.tar.bz2
+          tar xf ${CORE_ARTIFACT}.tar.bz2 # will create ArduinoCore-zephyr/
           echo "REPORT_FILE=$(echo ${FQBN} | tr ':' '-').json" >> $GITHUB_ENV
 
       - name: Create Blink sketch
@@ -209,8 +219,7 @@ jobs:
       id-token: write
       contents: read
     env:
-      CORE_ARTIFACT: ${{ needs.package-core.outputs.CORE_ARTIFACT }}
-      ARTIFACT_FILE: ${{ needs.package-core.outputs.CORE_ARTIFACT }}.tar.bz2
+      CORE_ARTIFACT: ArduinoCore-zephyr-${{ needs.package-core.outputs.CORE_HASH }}
     steps:
       - uses: actions/download-artifact@v4
         with:
@@ -223,7 +232,7 @@ jobs:
           aws-region: ${{ secrets.AWS_REGION }}
 
       - name: Upload artifact
-        run: aws s3 cp ${ARTIFACT_FILE} s3://${{ secrets.S3_BUCKET }}/
+        run: aws s3 cp ArduinoCore-*.tar.bz2 s3://${{ secrets.S3_BUCKET }}/
 
   publish-json:
     name: Publish json
@@ -232,12 +241,9 @@ jobs:
     needs:
       - package-core
       - test-core
-      - publish-core
     env:
-      CORE_ARTIFACT: ${{ needs.package-core.outputs.CORE_ARTIFACT }}
-      ARTIFACT_FILE: ${{ needs.package-core.outputs.CORE_ARTIFACT }}.tar.bz2
+      CORE_ARTIFACT: ArduinoCore-zephyr-${{ needs.package-core.outputs.CORE_HASH }}
       CORE_TAG: ${{ needs.package-core.outputs.CORE_TAG }}
-      PACKAGE_INDEX_JSON: zephyr-core-${{ needs.package-core.outputs.CORE_TAG }}.json
     steps:
       - uses: actions/checkout@v4
         with:
@@ -251,12 +257,15 @@ jobs:
         with:
           name: ${{ env.CORE_ARTIFACT }}
 
-        # uses: ARTIFACT_FILE CORE_TAG PACKAGE_INDEX_JSON
       - name: Prepare package index snippet
-        run: ./extra/gen_package_index_json.sh
+        run: |
+          PACKAGE=zephyr
+          ARTIFACT_FILE=ArduinoCore-${PACKAGE}-${CORE_HASH}.tar.bz2
+          PACKAGE_JSON=ArduinoCore-${PACKAGE}-${CORE_TAG}.json
+          ./extra/gen_package_index_json.sh ${CORE_TAG} ${ARTIFACT_FILE} > ${PACKAGE_JSON}
 
       - name: Archive package index snippet
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ env.PACKAGE_INDEX_JSON }}
-          path: ${{ env.PACKAGE_INDEX_JSON }}
+          name: ArduinoCore-zephyr-${{ env.CORE_TAG }}-jsons
+          path: ArduinoCore-*-${{ env.CORE_TAG }}.json
diff --git a/extra/bootstrap.sh b/extra/bootstrap.sh
@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -e
+
 if [ ! -f platform.txt ]; then
   echo Launch this script from the root core folder as ./extra/bootstrap.sh
   exit 2
diff --git a/extra/gen_package_index_json.sh b/extra/gen_package_index_json.sh
@@ -1,13 +1,22 @@
 #!/bin/bash
 
-if [ -z "$CORE_TAG" ]; then
-  echo "This script can be used in Github CI only."
+if [ $# -ne 2 ] ; then
+	echo "Usage: $0 <core_tag> <artifact_file>"
   exit 1
 fi
 
-cat extra/zephyr-core-template.json | sed \
+PACKAGE=zephyr
+CORE_TAG=$1
+ARTIFACT_FILE=$2
+
+if [ ! -f "$ARTIFACT_FILE" ] ; then
+	echo "Artifact file '$ARTIFACT_FILE' not found" 1>&2
+	exit 1
+fi
+
+JSON_TEMPLATE="extra/zephyr-core-template.json"
+cat $JSON_TEMPLATE | sed \
 	-e "s/__CORE_TAG__/$CORE_TAG/" \
-	-e "s/__ARTIFACT_FILE__/$ARTIFACT_FILE/" \
+	-e "s/__ARTIFACT_FILE__/$(basename $ARTIFACT_FILE)/" \
 	-e "s/__ARTIFACT_HASH__/$(sha256sum $ARTIFACT_FILE | awk '{print $1}')/" \
-	-e "s/__ARTIFACT_SIZE__/$(stat -c %s $ARTIFACT_FILE)/" \
-	> $PACKAGE_INDEX_JSON
+	-e "s/__ARTIFACT_SIZE__/$(stat -c %s $ARTIFACT_FILE)/" | jq .
diff --git a/extra/get_board_details.sh b/extra/get_board_details.sh
@@ -7,14 +7,15 @@ get_boards() {
 get_board_field() {
 	board=$1
 	field=$2
-	cat boards.txt | sed -e 's/#.*//' | grep -E "^$board\\.build\\.$field=" | cut -d '=' -f2- | sed -e 's/"/\"/g'
+	cat boards.txt | sed -e 's/#.*//' | grep -E "^$board\\.$field=" | cut -d '=' -f2- | sed -e 's/"/\"/g'
 }
 
 for BOARD in $(get_boards); do
-	VARIANT=$(get_board_field $BOARD "variant")
-	TARGET=$(get_board_field $BOARD "zephyr_target")
-	ARGS=$(get_board_field $BOARD "zephyr_args")
-	HALS=$(get_board_field $BOARD "zephyr_hals")
+	NAME=$(get_board_field $BOARD "name")
+	VARIANT=$(get_board_field $BOARD "build\\.variant")
+	TARGET=$(get_board_field $BOARD "build\\.zephyr_target")
+	ARGS=$(get_board_field $BOARD "build\\.zephyr_args")
+	HALS=$(get_board_field $BOARD "build\\.zephyr_hals")
 
 	if [ -z "$TARGET" ] ; then
 		echo "error: missing '$BOARD.build.zephyr_target'" 1>&2
@@ -25,5 +26,14 @@ for BOARD in $(get_boards); do
 		exit 1
 	fi
 
-	echo "{ \"board\": \"$BOARD\", \"variant\": \"$VARIANT\", \"target\": \"$TARGET\", \"args\": \"$ARGS\", \"hals\": \"$HALS\" }"
+	cat << EOF
+	{
+	  "name": "$NAME",
+	  "board": "$BOARD",
+	  "variant": "$VARIANT",
+	  "target": "$TARGET",
+	  "args": "$ARGS",
+	  "hals": "$HALS",
+	}
+EOF
 done | jq -crs .
diff --git a/extra/package_core.sh b/extra/package_core.sh
@@ -8,31 +8,40 @@ if [ -z "$1" ]; then
 fi
 
 if [ ! -f platform.txt ]; then
-	echo "Launch this script from the root core folder as ./extra/package_core.sh VERSION"
+	echo "Launch this script from the root core folder."
 	exit 2
 fi
 
-PACKAGE=ArduinoCore-zephyr
+PACKAGE=zephyr
 VERSION=$1
 OUTPUT_FILE=${2:-distrib/${PACKAGE}-${VERSION}.tar.bz2}
 
 # create a temporary platform.txt file with the correct version
 TEMP_PLATFORM=$(mktemp -p . | sed 's/\.\///')
 sed -e "s/^version=.*/version=${VERSION}/" platform.txt > ${TEMP_PLATFORM}
 
-TEMP_LIST=$(mktemp)
-echo ${TEMP_PLATFORM} > ${TEMP_LIST}
+declutter_file() {
+	[ -f "$1" ] || return 0
+	cat "$1" | sed -e 's/\s*#.*//' | grep -v '^\s*$'
+}
 
-# import a basic list of files and directories
-cat extra/package_core.inc | sed -e 's/\s*#.*//' | grep -v '^\s*$' >> ${TEMP_LIST}
-
-# add the board-specific files
+# create the list of files and directories to include
+TEMP_INC=$(mktemp -p . | sed 's/\.\///')
+echo ${TEMP_PLATFORM} >> ${TEMP_INC}
+declutter_file extra/package_core.inc >> ${TEMP_INC}
 extra/get_board_details.sh | jq -cr '.[]' | while read -r item; do
 	variant=$(jq -cr '.variant' <<< "$item")
-	echo "variants/${variant}/" >> ${TEMP_LIST}
-	ls firmwares/zephyr-${variant}.* >> ${TEMP_LIST}
+	echo "::info::`${variant}`"
+	echo "variants/${variant}/" >> ${TEMP_INC}
+	ls firmwares/zephyr-${variant}.* >> ${TEMP_INC}
 done
-cat ${TEMP_LIST}
+
+# create the list of files and directories to exclude
+TEMP_EXC=$(mktemp -p . | sed 's/\.\///')
+declutter_file extra/package.exc >> ${TEMP_EXC}
+
 mkdir -p $(dirname ${OUTPUT_FILE})
-tar -cjhf ${OUTPUT_FILE} -X extra/package_core.exc -T ${TEMP_LIST} --transform "s,${TEMP_PLATFORM},platform.txt," --transform "s,^,${PACKAGE}/,"
-rm -f ${TEMP_LIST} ${TEMP_PLATFORM}
+tar -cjhf ${OUTPUT_FILE} -X ${TEMP_EXC} -T ${TEMP_INC} \
+	--transform "s,${TEMP_PLATFORM},platform.txt," \
+	--transform "s,^,ArduinoCore-zephyr/,"
+rm -f ${TEMP_INC} ${TEMP_EXC} ${TEMP_PLATFORM}
