From 9ad2eb400f3da40ac1a28ee6853dcc2290df4fe2 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Sat, 25 Oct 2025 21:40:39 +0800 Subject: [PATCH 1/4] CI: Enhance common script with strict mode This adds bash strict mode, color output helpers, ASSERT function, and cleanup mechanism to improve CI script reliability and debugging. - Add color output functions (print_success, print_error, print_warning) - Add ASSERT function for test validation - Add cleanup function registry with trap handler - Fix POSIX compliance (use = instead of ==) --- .ci/common.sh | 74 ++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 67 insertions(+), 7 deletions(-) diff --git a/.ci/common.sh b/.ci/common.sh index 7300139b..1e0a964f 100644 --- a/.ci/common.sh +++ b/.ci/common.sh @@ -1,3 +1,8 @@ +# Bash strict mode (enabled only when executed directly, not sourced) +if ! (return 0 2> /dev/null); then + set -euo pipefail +fi + # Expect host is Linux/x86_64, Linux/aarch64, macOS/arm64 MACHINE_TYPE=$(uname -m) @@ -15,12 +20,67 @@ check_platform() esac } -if [[ "${OS_TYPE}" == "Linux" ]]; then +if [ "${OS_TYPE}" = "Linux" ]; then PARALLEL=-j$(nproc) else PARALLEL=-j$(sysctl -n hw.logicalcpu) fi +# Color output helpers +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +print_success() +{ + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +print_error() +{ + echo -e "${RED}[ERROR]${NC} $1" >&2 +} + +print_warning() +{ + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +# Assertion function for tests +# Usage: ASSERT +ASSERT() +{ + local condition=$1 + shift + local message="$*" + + if ! eval "${condition}"; then + print_error "Assertion failed: ${message}" + print_error "Condition: ${condition}" + return 1 + fi +} + +# Cleanup function registry +CLEANUP_FUNCS=() + +register_cleanup() +{ + CLEANUP_FUNCS+=("$1") +} + +cleanup() +{ + local func + for func in "${CLEANUP_FUNCS[@]-}"; do + [ -n "${func}" ] || continue + eval "${func}" || true + done +} + +trap cleanup EXIT + # Universal download utility with curl/wget compatibility # Provides consistent interface regardless of which tool is available @@ -49,7 +109,7 @@ download_to_stdout() local url="$1" case "$DOWNLOAD_TOOL" in curl) - curl -fsSL "$url" + curl -fS --retry 5 --retry-delay 2 --retry-max-time 60 -sL "$url" ;; wget) wget -qO- "$url" @@ -66,7 +126,7 @@ download_to_file() local output="$2" case "$DOWNLOAD_TOOL" in curl) - curl -fsSL -o "$output" "$url" + curl -fS --retry 5 --retry-delay 2 --retry-max-time 60 -sL -o "$output" "$url" ;; wget) wget -q -O "$output" "$url" @@ -88,7 +148,7 @@ download_with_headers() for header in "$@"; do headers+=(-H "$header") done - curl -fsSL "${headers[@]}" "$url" + curl -fS --retry 5 --retry-delay 2 --retry-max-time 60 -sL "${headers[@]}" "$url" ;; wget) for header in "$@"; do @@ -107,7 +167,7 @@ download_silent() local url="$1" case "$DOWNLOAD_TOOL" in curl) - curl -fsSL "$url" + curl -fS --retry 5 --retry-delay 2 --retry-max-time 60 -sL "$url" ;; wget) wget -qO- "$url" @@ -124,7 +184,7 @@ download_with_progress() local output="$2" case "$DOWNLOAD_TOOL" in curl) - curl -fL -# -o "$output" "$url" + curl -fS --retry 5 --retry-delay 2 --retry-max-time 60 -L -# -o "$output" "$url" ;; wget) wget -O "$output" "$url" @@ -141,7 +201,7 @@ check_url() local url="$1" case "$DOWNLOAD_TOOL" in curl) - curl -fsSL --head "$url" > /dev/null 2>&1 + curl -fS --retry 5 --retry-delay 2 --retry-max-time 60 -sL --head "$url" > /dev/null 2>&1 ;; wget) wget --spider -q "$url" 2> /dev/null From f4070d9d5bd492bde3eb619220582a8571f659a4 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Sat, 25 Oct 2025 21:40:49 +0800 Subject: [PATCH 2/4] CI: Simplify format checker This replaces temporary file generation with clang-format's built-in dry-run mode. - Use clang-format -n --Werror instead of diff with temp files - Use git ls-files to filter tracked files only - Use mapfile for bash 4+ compatibility - Capture exit codes properly for each checker --- .ci/check-format.sh | 75 ++++++++++++++++++++++++++++++--------------- 1 file changed, 51 insertions(+), 24 deletions(-) diff --git a/.ci/check-format.sh b/.ci/check-format.sh index 1d98571f..74503163 100755 --- a/.ci/check-format.sh +++ b/.ci/check-format.sh @@ -4,31 +4,58 @@ set -u -o pipefail -set -x - REPO_ROOT="$(git rev-parse --show-toplevel)" -C_SOURCES=$(find "${REPO_ROOT}" | egrep "\.(c|cxx|cpp|h|hpp)$") -for file in ${C_SOURCES}; do - clang-format-18 ${file} > expected-format - diff -u -p --label="${file}" --label="expected coding style" ${file} expected-format -done -C_MISMATCH_LINE_CNT=$(clang-format-18 --output-replacements-xml ${C_SOURCES} | egrep -c "") - -SH_SOURCES=$(find "${REPO_ROOT}" | egrep "\.sh$") -for file in ${SH_SOURCES}; do - shfmt -d "${file}" -done -SH_MISMATCH_FILE_CNT=$(shfmt -l ${SH_SOURCES}) - -PY_SOURCES=$(find "${REPO_ROOT}" | egrep "\.py$") -for file in ${PY_SOURCES}; do - echo "Checking Python file: ${file}" - black --diff "${file}" -done -PY_MISMATCH_FILE_CNT=0 -if [ -n "${PY_SOURCES}" ]; then - PY_MISMATCH_FILE_CNT=$(echo "$(black --check ${PY_SOURCES} 2>&1)" | grep -c "^would reformat ") +# Use git ls-files to exclude submodules and untracked files +C_SOURCES=() +while IFS= read -r file; do + [ -n "$file" ] && C_SOURCES+=("$file") +done < <(git ls-files -- '*.c' '*.cxx' '*.cpp' '*.h' '*.hpp') + +if [ ${#C_SOURCES[@]} -gt 0 ]; then + if command -v clang-format-18 > /dev/null 2>&1; then + echo "Checking C/C++ files..." + clang-format-18 -n --Werror "${C_SOURCES[@]}" + C_FORMAT_EXIT=$? + else + echo "Skipping C/C++ format check: clang-format-18 not found" >&2 + C_FORMAT_EXIT=0 + fi +else + C_FORMAT_EXIT=0 +fi + +SH_SOURCES=() +while IFS= read -r file; do + [ -n "$file" ] && SH_SOURCES+=("$file") +done < <(git ls-files -- '*.sh') + +if [ ${#SH_SOURCES[@]} -gt 0 ]; then + echo "Checking shell scripts..." + MISMATCHED_SH=$(shfmt -l "${SH_SOURCES[@]}") + if [ -n "$MISMATCHED_SH" ]; then + echo "The following shell scripts are not formatted correctly:" + printf '%s\n' "$MISMATCHED_SH" + shfmt -d "${SH_SOURCES[@]}" + SH_FORMAT_EXIT=1 + else + SH_FORMAT_EXIT=0 + fi +else + SH_FORMAT_EXIT=0 +fi + +PY_SOURCES=() +while IFS= read -r file; do + [ -n "$file" ] && PY_SOURCES+=("$file") +done < <(git ls-files -- '*.py') + +if [ ${#PY_SOURCES[@]} -gt 0 ]; then + echo "Checking Python files..." + black --check --diff "${PY_SOURCES[@]}" + PY_FORMAT_EXIT=$? +else + PY_FORMAT_EXIT=0 fi -exit $((C_MISMATCH_LINE_CNT + SH_MISMATCH_FILE_CNT + PY_MISMATCH_FILE_CNT)) +exit $((C_FORMAT_EXIT + SH_FORMAT_EXIT + PY_FORMAT_EXIT)) From bf0ecdc9ee80393c88771d8300d1da0ff52a5b1c Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Sat, 25 Oct 2025 21:40:59 +0800 Subject: [PATCH 3/4] CI: Fix POSIX compliance This replaces bash-specific syntax with POSIX-compliant alternatives for better portability across different shell environments. --- .ci/boot-linux-prepare.sh | 8 ++-- .ci/boot-linux.sh | 75 ++++++++++++++++++++++++++++------ .ci/gdbstub-test.sh | 4 +- .ci/riscv-toolchain-install.sh | 4 +- 4 files changed, 71 insertions(+), 20 deletions(-) diff --git a/.ci/boot-linux-prepare.sh b/.ci/boot-linux-prepare.sh index 199c1cb2..0e801dbb 100755 --- a/.ci/boot-linux-prepare.sh +++ b/.ci/boot-linux-prepare.sh @@ -16,10 +16,10 @@ which mkfs.ext4 > /dev/null 2>&1 || which $(brew --prefix e2fsprogs)/sbin/mkfs.e echo "Error: mkfs.ext4 not found" exit 1 } -which 7z > /dev/null 2>&1 || { - echo "Error: 7z not found" - exit 1 -} +# Optional tooling used later in the test suite +if ! command -v debugfs > /dev/null 2>&1 && ! command -v 7z > /dev/null 2>&1; then + print_warning "Neither debugfs nor 7z is available; virtio-blk verification will be skipped." +fi ACTION=$1 diff --git a/.ci/boot-linux.sh b/.ci/boot-linux.sh index 4d8161ee..c2aafc20 100755 --- a/.ci/boot-linux.sh +++ b/.ci/boot-linux.sh @@ -6,13 +6,54 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" check_platform -function cleanup +cleanup() { sleep 1 pkill -9 rv32emu } -function ASSERT +check_image_for_file() +{ + local image_path=$1 + local file_path=$2 + local tool_available=0 + local debugfs_cmd + + debugfs_cmd=$(command -v debugfs 2> /dev/null || true) + if [ -z "${debugfs_cmd}" ] && [ -x /sbin/debugfs ]; then + debugfs_cmd=/sbin/debugfs + fi + + if [ -n "${debugfs_cmd}" ]; then + tool_available=1 + if "${debugfs_cmd}" -R "stat ${file_path}" "${image_path}" > /dev/null 2>&1; then + return 0 + fi + fi + + if command -v 7z > /dev/null 2>&1; then + tool_available=1 + if 7z l "${image_path}" | grep -q "${file_path}"; then + return 0 + fi + fi + + if [ -n "${debugfs_cmd}" ]; then + tool_available=1 + if sudo "${debugfs_cmd}" -R "stat ${file_path}" "${image_path}" > /dev/null 2>&1; then + return 0 + fi + fi + + if [ "${tool_available}" -eq 0 ]; then + print_warning "Skipping verification of ${file_path} in ${image_path}: neither debugfs nor 7z is available." + return 0 + fi + + return 1 +} + +ASSERT() { $* local RES=$? @@ -130,9 +171,12 @@ for i in "${!TEST_OPTIONS[@]}"; do OPTS="${OPTS_BASE}" # No need to add option when running base test - if [[ ! "${TEST_OPTIONS[$i]}" =~ "base" ]]; then - OPTS+="${TEST_OPTIONS[$i]}" - fi + case "${TEST_OPTIONS[$i]}" in + *base*) ;; + *) + OPTS+="${TEST_OPTIONS[$i]}" + ;; + esac RUN_LINUX="build/rv32emu ${OPTS}" ASSERT expect <<- DONE @@ -145,13 +189,20 @@ for i in "${!TEST_OPTIONS[@]}"; do cleanup printf "\nBoot Linux Test: [ ${MESSAGES[$ret]}${COLOR_N} ]\n" - if [[ "${TEST_OPTIONS[$i]}" =~ vblk ]]; then - # read-only test first, so the emu.txt definitely does not exist, skipping the check - if [[ ! "${TEST_OPTIONS[$i]}" =~ readonly ]]; then - 7z l ${VBLK_IMG} | grep emu.txt > /dev/null 2>&1 || ret=4 - fi - printf "Virtio-blk Test: [ ${MESSAGES[$ret]}${COLOR_N} ]\n" - fi + case "${TEST_OPTIONS[$i]}" in + *vblk*) + # read-only test first, so the emu.txt definitely does not exist, skipping the check + case "${TEST_OPTIONS[$i]}" in + *readonly*) ;; + *) + if ! check_image_for_file "${VBLK_IMG}" emu.txt; then + ret=4 + fi + ;; + esac + printf "Virtio-blk Test: [ ${MESSAGES[$ret]}${COLOR_N} ]\n" + ;; + esac done exit ${ret} diff --git a/.ci/gdbstub-test.sh b/.ci/gdbstub-test.sh index 156aa558..5cba5770 100755 --- a/.ci/gdbstub-test.sh +++ b/.ci/gdbstub-test.sh @@ -9,8 +9,8 @@ prefixes=("${CROSS_COMPILE}" "riscv32-unknown-elf-" "riscv-none-elf-") for prefix in "${prefixes[@]}"; do utility=${prefix}gdb set +e # temporarily disable exit on error - command -v "${utility}" &> /dev/null - if [[ $? == 0 ]]; then + command -v "${utility}" > /dev/null 2>&1 + if [ $? = 0 ]; then GDB=${utility} fi set -e diff --git a/.ci/riscv-toolchain-install.sh b/.ci/riscv-toolchain-install.sh index 9f0a40d6..b6faab90 100755 --- a/.ci/riscv-toolchain-install.sh +++ b/.ci/riscv-toolchain-install.sh @@ -9,11 +9,11 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" check_platform mkdir -p toolchain -if [[ "$#" == "0" ]] || [[ "$1" != "riscv-collab" ]]; then +if [ "$#" = "0" ] || [ "$1" != "riscv-collab" ]; then GCC_VER=15.2.0-1 TOOLCHAIN_REPO=https://github.com/xpack-dev-tools/riscv-none-elf-gcc-xpack - if [[ "${OS_TYPE}" == "Linux" ]]; then + if [ "${OS_TYPE}" = "Linux" ]; then case "${MACHINE_TYPE}" in "x86_64") TOOLCHAIN_URL=${TOOLCHAIN_REPO}/releases/download/v${GCC_VER}/xpack-riscv-none-elf-gcc-${GCC_VER}-linux-x64.tar.gz From 16862b0468a9d8e98e07322705581b0420a87a5d Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Mon, 27 Oct 2025 23:27:37 +0800 Subject: [PATCH 4/4] CI: Pin Python 3.12 on macOS --- .github/workflows/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 50a8e9b3..4cb3308b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -345,6 +345,10 @@ jobs: .ci/riscv-toolchain-install.sh echo "${{ github.workspace }}/toolchain/bin" >> $GITHUB_PATH echo "$(brew --prefix llvm@18)/bin" >> $GITHUB_PATH + - name: Set up Python 3.12 + uses: actions/setup-python@v6 + with: + python-version: '3.12' - name: Install compiler id: install_cc uses: rlalik/setup-cpp-compiler@master