From 3f079306d39734e60e3557eca4c156f90e39e942 Mon Sep 17 00:00:00 2001 From: Daniel Ching Date: Wed, 30 Apr 2025 13:09:20 -0500 Subject: [PATCH 01/24] CI: Install latest compute-sanitizer separately from CTK --- .github/actions/fetch_ctk/action.yml | 2 +- .github/workflows/test-wheel-linux.yml | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/actions/fetch_ctk/action.yml b/.github/actions/fetch_ctk/action.yml index eee8efb31..96a16d086 100644 --- a/.github/actions/fetch_ctk/action.yml +++ b/.github/actions/fetch_ctk/action.yml @@ -17,7 +17,7 @@ inputs: description: "A list of the CTK components to install as a comma-separated list. e.g. 'cuda_nvcc,cuda_nvrtc,cuda_cudart'" required: false type: string - default: "cuda_nvcc,cuda_cudart,cuda_nvrtc,cuda_profiler_api,cuda_cccl,cuda_sanitizer_api,libnvjitlink" + default: "cuda_nvcc,cuda_cudart,cuda_nvrtc,cuda_profiler_api,cuda_cccl,libnvjitlink" runs: using: composite diff --git a/.github/workflows/test-wheel-linux.yml b/.github/workflows/test-wheel-linux.yml index 83dad5cec..ebb9f22e1 100644 --- a/.github/workflows/test-wheel-linux.yml +++ b/.github/workflows/test-wheel-linux.yml @@ -184,6 +184,15 @@ jobs: host-platform: ${{ inputs.host-platform }} cuda-version: ${{ inputs.cuda-version }} + - name: Set up latest cuda_sanitizer_api + if: ${{ inputs.local-ctk == '1' }} + uses: ./.github/actions/fetch_ctk + continue-on-error: false + with: + host-platform: ${{ inputs.host-platform }} + cuda-version: "12.8.1" + cuda-components: "cuda_sanitizer_api" + - name: Set up compute-sanitizer run: | # We don't test compute-sanitizer on CTK<12 because backporting fixes is too much effort From 9aa44f927253fad5c7d77a35a2ccc696b6d3edac Mon Sep 17 00:00:00 2001 From: Daniel Ching <9604511+carterbox@users.noreply.github.com> Date: Wed, 30 Apr 2025 14:41:40 -0500 Subject: [PATCH 02/24] Use mkdir -p to reuse CUDA_PATH --- .github/actions/fetch_ctk/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/fetch_ctk/action.yml b/.github/actions/fetch_ctk/action.yml index 96a16d086..c3cd7aae8 100644 --- a/.github/actions/fetch_ctk/action.yml +++ b/.github/actions/fetch_ctk/action.yml @@ -51,7 +51,7 @@ runs: shell: bash --noprofile --norc -xeuo pipefail {0} run: | CUDA_PATH="./cuda_toolkit" - mkdir $CUDA_PATH + mkdir -p $CUDA_PATH # The binary archives (redist) are guaranteed to be updated as part of the release posting. CTK_BASE_URL="https://developer.download.nvidia.com/compute/cuda/redist/" From 10164115526964767e8e54534b9538d6d7932ae0 Mon Sep 17 00:00:00 2001 From: Daniel Ching Date: Thu, 1 May 2025 11:33:22 -0500 Subject: [PATCH 03/24] BUG: Always start with clean CUDA_PATH when building cache --- .github/actions/fetch_ctk/action.yml | 3 ++- .github/actions/fetch_ctk/guess_latest.sh | 28 +++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 .github/actions/fetch_ctk/guess_latest.sh diff --git a/.github/actions/fetch_ctk/action.yml b/.github/actions/fetch_ctk/action.yml index c3cd7aae8..31f1670f6 100644 --- a/.github/actions/fetch_ctk/action.yml +++ b/.github/actions/fetch_ctk/action.yml @@ -51,7 +51,8 @@ runs: shell: bash --noprofile --norc -xeuo pipefail {0} run: | CUDA_PATH="./cuda_toolkit" - mkdir -p $CUDA_PATH + rm -f $CUDA_PATH + mkdir $CUDA_PATH # The binary archives (redist) are guaranteed to be updated as part of the release posting. CTK_BASE_URL="https://developer.download.nvidia.com/compute/cuda/redist/" diff --git a/.github/actions/fetch_ctk/guess_latest.sh b/.github/actions/fetch_ctk/guess_latest.sh new file mode 100644 index 000000000..0ffdaac07 --- /dev/null +++ b/.github/actions/fetch_ctk/guess_latest.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# URL to search +URL="https://developer.download.nvidia.com/compute/cuda/redist/" + +# Fetch the directory listing and extract the latest version number +get_latest_version() { + # Get the HTML content of the page + local html_content=$(wget -q -O - "$URL") + + # Extract links matching the pattern redistrib_?.?.?.json + local files=$(echo "$html_content" | grep -oP 'redistrib_[0-9]+\.[0-9]+\.[0-9]+\.json' | cut -d'"' -f2) + + # If files were found, extract the version numbers and find the latest + if [ -n "$files" ]; then + # Extract just the version numbers using regex + local versions=$(echo "$files" | grep -oP 'redistrib_\K[0-9]+\.[0-9]+\.[0-9]+(?=\.json)') + + # Sort the versions and get the latest + local latest_version=$(echo "$versions" | sort -V | tail -n 1) + echo "$latest_version" + else + echo "No files matching the pattern were found." + return 1 + fi +} + +# Call the function and store the result +latest_version=$(get_latest_version) From 6e917f24d63aa5cc8591e3f38069d3fe015aaaa2 Mon Sep 17 00:00:00 2001 From: Daniel Ching Date: Thu, 1 May 2025 11:44:05 -0500 Subject: [PATCH 04/24] REF: Setup CUDA component cache in TMP directory Prevent collision with components already installed to the CUDA_PATH --- .github/actions/fetch_ctk/action.yml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/actions/fetch_ctk/action.yml b/.github/actions/fetch_ctk/action.yml index 31f1670f6..670250220 100644 --- a/.github/actions/fetch_ctk/action.yml +++ b/.github/actions/fetch_ctk/action.yml @@ -50,9 +50,9 @@ runs: if: ${{ steps.ctk-get-cache.outputs.cache-hit != 'true' }} shell: bash --noprofile --norc -xeuo pipefail {0} run: | - CUDA_PATH="./cuda_toolkit" - rm -f $CUDA_PATH - mkdir $CUDA_PATH + CACHE_TMP_DIR="./cache_tmp_dir" + rm -f $CACHE_TMP_DIR + mkdir $CACHE_TMP_DIR # The binary archives (redist) are guaranteed to be updated as part of the release posting. CTK_BASE_URL="https://developer.download.nvidia.com/compute/cuda/redist/" @@ -64,14 +64,14 @@ runs: CTK_SUBDIR="linux-sbsa" fi function extract() { - tar -xvf $1 -C $CUDA_PATH --strip-components=1 + tar -xvf $1 -C $CACHE_TMP_DIR --strip-components=1 } elif [[ "${{ inputs.host-platform }}" == "win-64" ]]; then CTK_SUBDIR="windows-x86_64" function extract() { _TEMP_DIR_=$(mktemp -d) unzip $1 -d $_TEMP_DIR_ - cp -r $_TEMP_DIR_/*/* $CUDA_PATH + cp -r $_TEMP_DIR_/*/* $CACHE_TMP_DIR rm -rf $_TEMP_DIR_ } fi @@ -100,11 +100,15 @@ runs: for item in $(echo $CTK_CACHE_COMPONENTS | tr ',' ' '); do populate_cuda_path "$item" done - ls -l $CUDA_PATH + ls -l $CACHE_TMP_DIR # Prepare the cache # Note: try to escape | and > ... - tar -czvf ${CTK_CACHE_FILENAME} ${CUDA_PATH} + tar -czvf ${CTK_CACHE_FILENAME} ${CACHE_TMP_DIR} + + CUDA_PATH="./cuda_toolkit" + mkdir -p $CUDA_PATH + mv $CACHE_TMP_DIR/* $CUDA_PATH - name: Upload CTK cache if: ${{ always() && From 6b02f1a1e89453ba29b5755e05ffde35bdd736cc Mon Sep 17 00:00:00 2001 From: Daniel Ching Date: Thu, 1 May 2025 11:46:28 -0500 Subject: [PATCH 05/24] DEV: Skip non python-3.12 for debugging --- .github/workflows/build-and-test.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index a00a94028..2e2f4da1c 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -25,14 +25,14 @@ jobs: matrix: host-platform: - linux-64 - - linux-aarch64 - - win-64 + # - linux-aarch64 + # - win-64 python-version: - - "3.13" + # - "3.13" - "3.12" - - "3.11" - - "3.10" - - "3.9" + # - "3.11" + # - "3.10" + # - "3.9" cuda-version: # Note: this is for build-time only. - "12.8.0" @@ -63,11 +63,11 @@ jobs: uses: actions/setup-python@v5 with: python-version: "3.12" - + - name: Set up MSVC if: ${{ startsWith(matrix.host-platform, 'win') }} uses: ilammy/msvc-dev-cmd@v1 - + - name: Set environment variables run: | PYTHON_VERSION_FORMATTED=$(echo '${{ matrix.python-version }}' | tr -d '.') @@ -79,7 +79,7 @@ jobs: PWD=$(pwd) REPO_DIR=$(cygpath -w $PWD) fi - + echo "CUDA_BINDINGS_PARALLEL_LEVEL=$(nproc)" >> $GITHUB_ENV CUDA_CORE_ARTIFACT_BASENAME="cuda-core-python${PYTHON_VERSION_FORMATTED}-${{ matrix.host-platform }}" echo "CUDA_CORE_ARTIFACT_BASENAME=${CUDA_CORE_ARTIFACT_BASENAME}" >> $GITHUB_ENV @@ -215,13 +215,13 @@ jobs: matrix: host-platform: - linux-64 - - linux-aarch64 + # - linux-aarch64 python-version: - - "3.13" + # - "3.13" - "3.12" - - "3.11" - - "3.10" - - "3.9" + # - "3.11" + # - "3.10" + # - "3.9" cuda-version: # Note: this is for test-time only. - "12.8.0" From 5cf0f552ece1eb68cc149e9e3a9ea39021366f5c Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 1 May 2025 22:20:56 -0400 Subject: [PATCH 06/24] fix --- .github/actions/fetch_ctk/action.yml | 3 ++- .github/actions/fetch_ctk/guess_latest.sh | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/actions/fetch_ctk/action.yml b/.github/actions/fetch_ctk/action.yml index e8a5fc8f2..cb8b25e59 100644 --- a/.github/actions/fetch_ctk/action.yml +++ b/.github/actions/fetch_ctk/action.yml @@ -51,7 +51,7 @@ runs: shell: bash --noprofile --norc -xeuo pipefail {0} run: | CACHE_TMP_DIR="./cache_tmp_dir" - rm -f $CACHE_TMP_DIR + rm -rf $CACHE_TMP_DIR mkdir $CACHE_TMP_DIR # The binary archives (redist) are guaranteed to be updated as part of the release posting. @@ -120,6 +120,7 @@ runs: CUDA_PATH="./cuda_toolkit" mkdir -p $CUDA_PATH mv $CACHE_TMP_DIR/* $CUDA_PATH + rmdir $CACHE_TMP_DIR - name: Upload CTK cache if: ${{ always() && diff --git a/.github/actions/fetch_ctk/guess_latest.sh b/.github/actions/fetch_ctk/guess_latest.sh index 0ffdaac07..c6cec942d 100644 --- a/.github/actions/fetch_ctk/guess_latest.sh +++ b/.github/actions/fetch_ctk/guess_latest.sh @@ -1,4 +1,8 @@ #!/bin/bash +# Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED. +# +# SPDX-License-Identifier: Apache-2.0 + # URL to search URL="https://developer.download.nvidia.com/compute/cuda/redist/" From 247f2a0fd282cde42975532abf3d66db8696685d Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Fri, 2 May 2025 02:26:58 +0000 Subject: [PATCH 07/24] Revert "Merge pull request #593 from carterbox/dching/latest-sanitizer" This reverts commit bd770e14e0fb960b64862743eafc88394fb6f14a, reversing changes made to 19df0d93b0a0ce2c53262103a7742a1113724220. --- .github/actions/fetch_ctk/action.yml | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/.github/actions/fetch_ctk/action.yml b/.github/actions/fetch_ctk/action.yml index cb8b25e59..7afa82317 100644 --- a/.github/actions/fetch_ctk/action.yml +++ b/.github/actions/fetch_ctk/action.yml @@ -56,6 +56,7 @@ runs: # The binary archives (redist) are guaranteed to be updated as part of the release posting. CTK_BASE_URL="https://developer.download.nvidia.com/compute/cuda/redist/" + CTK_JSON_URL="$CTK_BASE_URL/redistrib_${{ inputs.cuda-version }}.json" if [[ "${{ inputs.host-platform }}" == linux* ]]; then if [[ "${{ inputs.host-platform }}" == "linux-64" ]]; then CTK_SUBDIR="linux-x86_64" @@ -74,15 +75,13 @@ runs: rm -rf $_TEMP_DIR_ } fi - function populate_cuda_path() { # take the component name as a argument function download() { curl -kLSs $1 -o $2 } - local CTK_COMPONENT=$1 - local CTK_VERSION=$2 - CTK_COMPONENT_REL_PATH="$(curl -s ${CTK_BASE_URL}/redistrib_${CTK_VERSION}.json | + CTK_COMPONENT=$1 + CTK_COMPONENT_REL_PATH="$(curl -s $CTK_JSON_URL | python -c "import sys, json; print(json.load(sys.stdin)['${CTK_COMPONENT}']['${CTK_SUBDIR}']['relative_path'])")" CTK_COMPONENT_URL="${CTK_BASE_URL}/${CTK_COMPONENT_REL_PATH}" CTK_COMPONENT_COMPONENT_FILENAME="$(basename $CTK_COMPONENT_REL_PATH)" @@ -99,17 +98,7 @@ runs: CTK_CACHE_COMPONENTS="${CTK_CACHE_COMPONENTS//,,/,}" # Get headers and shared libraries in place for item in $(echo $CTK_CACHE_COMPONENTS | tr ',' ' '); do - ctk_version="${{ inputs.cuda-version }}" - if [[ "$item" == "cuda_sanitizer_api" ]]; then - # Always use latest CTK for cuda_sanitizer_api - # FIXME: Automatically track latest CTK version - CUDA_MAJOR="$(cut -d '.' -f 1 <<< ${{ inputs.cuda-version }})" - if [[ "$CUDA_MAJOR" == "12" ]]; then - # TODO: Automatically track latest CTK minor version - ctk_version="12.8.0" - fi - fi - populate_cuda_path "$item" "$ctk_version" + populate_cuda_path "$item" done ls -l $CACHE_TMP_DIR From 6ff73650813c63bef121ea0f8a85ac0ee114eeb0 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Fri, 2 May 2025 02:47:15 +0000 Subject: [PATCH 08/24] use guess_latest.sh to get the latest CTK ver --- .github/actions/fetch_ctk/guess_latest.sh | 3 ++- .github/workflows/test-wheel-linux.yml | 20 ++++++++++++++------ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/.github/actions/fetch_ctk/guess_latest.sh b/.github/actions/fetch_ctk/guess_latest.sh index c6cec942d..dc4c4649a 100644 --- a/.github/actions/fetch_ctk/guess_latest.sh +++ b/.github/actions/fetch_ctk/guess_latest.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED. +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED. # # SPDX-License-Identifier: Apache-2.0 @@ -30,3 +30,4 @@ get_latest_version() { # Call the function and store the result latest_version=$(get_latest_version) +echo $latest_version diff --git a/.github/workflows/test-wheel-linux.yml b/.github/workflows/test-wheel-linux.yml index ebb9f22e1..24a40d870 100644 --- a/.github/workflows/test-wheel-linux.yml +++ b/.github/workflows/test-wheel-linux.yml @@ -78,6 +78,17 @@ jobs: fi fi + # We don't test compute-sanitizer on CTK<12 because backporting fixes is too much effort + # We only test compute-sanitizer on python 3.12 arbitrarily; we don't need to use sanitizer on the entire matrix + # Only local ctk installs have compute-sanitizer; there is not wheel for it + if [[ "${{ inputs.python-version }}" == "3.12" && "${{ inputs.cuda-version }}" != "11.8.0" && "${{ inputs.local-ctk }}" == 1 ]]; then + SETUP_SANITIZER=1 + echo "LATEST_CUDA_VERSION=$(bash .github/actions/fetch_ctk/guess_latest.sh)" >> $GITHUB_ENV + else + SETUP_SANITIZER=0 + fi + echo "SETUP_SANITIZER=${SETUP_SANITIZER}" >> $GITHUB_ENV + # make outputs from the previous job as env vars CUDA_CORE_ARTIFACT_BASENAME="cuda-core-python${PYTHON_VERSION_FORMATTED}-${{ inputs.host-platform }}" echo "PYTHON_VERSION_FORMATTED=${PYTHON_VERSION_FORMATTED}" >> $GITHUB_ENV @@ -185,20 +196,17 @@ jobs: cuda-version: ${{ inputs.cuda-version }} - name: Set up latest cuda_sanitizer_api - if: ${{ inputs.local-ctk == '1' }} + if: ${{ env.SETUP_SANITIZER == '1' }} uses: ./.github/actions/fetch_ctk continue-on-error: false with: host-platform: ${{ inputs.host-platform }} - cuda-version: "12.8.1" + cuda-version: ${{ env.LATEST_CUDA_VERSION }} cuda-components: "cuda_sanitizer_api" - name: Set up compute-sanitizer run: | - # We don't test compute-sanitizer on CTK<12 because backporting fixes is too much effort - # We only test compute-sanitizer on python 3.12 arbitrarily; we don't need to use sanitizer on the entire matrix - # Only local ctk installs have compute-sanitizer; there is not wheel for it - if [[ "${{ inputs.python-version }}" == "3.12" && "${{ inputs.cuda-version }}" != "11.8.0" && "${{ inputs.local-ctk }}" == 1 ]]; then + if [[ "${SETUP_SANITIZER}" == 1 ]]; then COMPUTE_SANITIZER="${CUDA_HOME}/bin/compute-sanitizer" COMPUTE_SANITIZER_VERSION=$(${COMPUTE_SANITIZER} --version | grep -Eo "[0-9]{4}\.[0-9]\.[0-9]" | sed -e 's/\.//g') SANITIZER_CMD="${COMPUTE_SANITIZER} --target-processes=all --launch-timeout=0 --tool=memcheck --error-exitcode=1" From a2d55d8018afa11dc285b55d7bb0aecf5181c968 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Fri, 2 May 2025 03:56:13 +0000 Subject: [PATCH 09/24] fix cache restoration --- .github/actions/fetch_ctk/action.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/actions/fetch_ctk/action.yml b/.github/actions/fetch_ctk/action.yml index 7afa82317..8224ef4be 100644 --- a/.github/actions/fetch_ctk/action.yml +++ b/.github/actions/fetch_ctk/action.yml @@ -124,8 +124,11 @@ runs: shell: bash --noprofile --norc -xeuo pipefail {0} run: | ls -l + CACHE_TMP_DIR="./cache_tmp_dir" CUDA_PATH="./cuda_toolkit" tar -xzvf $CTK_CACHE_FILENAME + mv $CACHE_TMP_DIR/* $CUDA_PATH + rmdir $CACHE_TMP_DIR ls -l $CUDA_PATH if [ ! -d "$CUDA_PATH/include" ]; then exit 1 From 1a158698ac60f6445d278ec15743b638d398339f Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Fri, 2 May 2025 04:04:24 +0000 Subject: [PATCH 10/24] fix cache restoration --- .github/actions/fetch_ctk/action.yml | 5 +++-- .github/workflows/install_gpu_driver.ps1 | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/actions/fetch_ctk/action.yml b/.github/actions/fetch_ctk/action.yml index 8224ef4be..9f33d9cfd 100644 --- a/.github/actions/fetch_ctk/action.yml +++ b/.github/actions/fetch_ctk/action.yml @@ -107,7 +107,7 @@ runs: tar -czvf ${CTK_CACHE_FILENAME} ${CACHE_TMP_DIR} CUDA_PATH="./cuda_toolkit" - mkdir -p $CUDA_PATH + mkdir $CUDA_PATH mv $CACHE_TMP_DIR/* $CUDA_PATH rmdir $CACHE_TMP_DIR @@ -125,8 +125,9 @@ runs: run: | ls -l CACHE_TMP_DIR="./cache_tmp_dir" - CUDA_PATH="./cuda_toolkit" tar -xzvf $CTK_CACHE_FILENAME + CUDA_PATH="./cuda_toolkit" + mkdir $CUDA_PATH mv $CACHE_TMP_DIR/* $CUDA_PATH rmdir $CACHE_TMP_DIR ls -l $CUDA_PATH diff --git a/.github/workflows/install_gpu_driver.ps1 b/.github/workflows/install_gpu_driver.ps1 index 980e64996..955a304db 100644 --- a/.github/workflows/install_gpu_driver.ps1 +++ b/.github/workflows/install_gpu_driver.ps1 @@ -1,4 +1,6 @@ -#Requires -RunAsAdministrator +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED. +# +# SPDX-License-Identifier: Apache-2.0 # Install the driver function Install-Driver { @@ -23,7 +25,7 @@ function Install-Driver { $ProgressPreference = $ProgressPreference_tmp Write-Output 'Download complete!' - # Install the file with the specified path from earlier as well as the RunAs admin option + # Install the file with the specified path from earlier Write-Output 'Running the driver installer...' Start-Process -FilePath $file_dir -ArgumentList $install_args -Wait Write-Output 'Done!' From d49e52948e647c99764808d17af7cd339b1f808b Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Fri, 2 May 2025 04:28:26 +0000 Subject: [PATCH 11/24] guess_latest needs wget --- .github/workflows/test-wheel-linux.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test-wheel-linux.yml b/.github/workflows/test-wheel-linux.yml index 24a40d870..34520fb58 100644 --- a/.github/workflows/test-wheel-linux.yml +++ b/.github/workflows/test-wheel-linux.yml @@ -52,6 +52,14 @@ jobs: with: fetch-depth: 0 + - name: Install dependencies + uses: ./.github/actions/install_unix_deps + continue-on-error: false + with: + # gcc for Cython tests, jq/wget for artifact fetching + dependencies: "build-essential jq wget" + dependent_exes: "gcc jq wget" + - name: Set environment variables run: | PYTHON_VERSION_FORMATTED=$(echo '${{ inputs.python-version }}' | tr -d '.') @@ -102,14 +110,6 @@ jobs: echo "SKIP_CUDA_BINDINGS_TEST=${SKIP_CUDA_BINDINGS_TEST}" >> $GITHUB_ENV echo "SKIP_CUDA_CORE_CYTHON_TEST=${SKIP_CUDA_CORE_CYTHON_TEST}" >> $GITHUB_ENV - - name: Install dependencies - uses: ./.github/actions/install_unix_deps - continue-on-error: false - with: - # gcc for Cython tests, jq/wget for artifact fetching - dependencies: "build-essential jq wget" - dependent_exes: "gcc jq wget" - - name: Download cuda-python build artifacts if: ${{ env.SKIP_CUDA_BINDINGS_TEST == '0'}} uses: actions/download-artifact@v4 From d9a2d6692b0cd4cf3161b31db8f9827ac3f3f7bd Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Fri, 2 May 2025 00:53:42 -0400 Subject: [PATCH 12/24] dunno what I was thinking --- .github/actions/fetch_ctk/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/fetch_ctk/action.yml b/.github/actions/fetch_ctk/action.yml index 9f33d9cfd..0fd98c7b9 100644 --- a/.github/actions/fetch_ctk/action.yml +++ b/.github/actions/fetch_ctk/action.yml @@ -107,7 +107,7 @@ runs: tar -czvf ${CTK_CACHE_FILENAME} ${CACHE_TMP_DIR} CUDA_PATH="./cuda_toolkit" - mkdir $CUDA_PATH + mkdir -p $CUDA_PATH mv $CACHE_TMP_DIR/* $CUDA_PATH rmdir $CACHE_TMP_DIR @@ -127,7 +127,7 @@ runs: CACHE_TMP_DIR="./cache_tmp_dir" tar -xzvf $CTK_CACHE_FILENAME CUDA_PATH="./cuda_toolkit" - mkdir $CUDA_PATH + mkdir -p $CUDA_PATH mv $CACHE_TMP_DIR/* $CUDA_PATH rmdir $CACHE_TMP_DIR ls -l $CUDA_PATH From 2a6edc29231c890c8132a8cdedee1e5455576445 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Fri, 2 May 2025 01:22:13 -0400 Subject: [PATCH 13/24] not sure if this would work on git for windows... --- .github/actions/fetch_ctk/action.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/actions/fetch_ctk/action.yml b/.github/actions/fetch_ctk/action.yml index 0fd98c7b9..a469a7232 100644 --- a/.github/actions/fetch_ctk/action.yml +++ b/.github/actions/fetch_ctk/action.yml @@ -124,12 +124,8 @@ runs: shell: bash --noprofile --norc -xeuo pipefail {0} run: | ls -l - CACHE_TMP_DIR="./cache_tmp_dir" - tar -xzvf $CTK_CACHE_FILENAME CUDA_PATH="./cuda_toolkit" - mkdir -p $CUDA_PATH - mv $CACHE_TMP_DIR/* $CUDA_PATH - rmdir $CACHE_TMP_DIR + tar -xzvf $CTK_CACHE_FILENAME -C $CUDA_PATH --strip-components=1 ls -l $CUDA_PATH if [ ! -d "$CUDA_PATH/include" ]; then exit 1 From 2eff1aa35c417215ee08c5f8125f77349dfb741a Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Fri, 2 May 2025 05:35:57 +0000 Subject: [PATCH 14/24] fix --- .github/actions/fetch_ctk/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/fetch_ctk/action.yml b/.github/actions/fetch_ctk/action.yml index a469a7232..4488747bb 100644 --- a/.github/actions/fetch_ctk/action.yml +++ b/.github/actions/fetch_ctk/action.yml @@ -125,6 +125,7 @@ runs: run: | ls -l CUDA_PATH="./cuda_toolkit" + mkdir -p $CUDA_PATH tar -xzvf $CTK_CACHE_FILENAME -C $CUDA_PATH --strip-components=1 ls -l $CUDA_PATH if [ ! -d "$CUDA_PATH/include" ]; then From 287302b5a0c6f5acba740aa5f8ab9f67b6c97e7d Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Fri, 2 May 2025 08:54:09 -0400 Subject: [PATCH 15/24] give rsync a shot --- .github/actions/fetch_ctk/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/fetch_ctk/action.yml b/.github/actions/fetch_ctk/action.yml index 4488747bb..b076e880e 100644 --- a/.github/actions/fetch_ctk/action.yml +++ b/.github/actions/fetch_ctk/action.yml @@ -108,8 +108,8 @@ runs: CUDA_PATH="./cuda_toolkit" mkdir -p $CUDA_PATH - mv $CACHE_TMP_DIR/* $CUDA_PATH - rmdir $CACHE_TMP_DIR + rsync -a $CACHE_TMP_DIR $CUDA_PATH + rm -rf $CACHE_TMP_DIR - name: Upload CTK cache if: ${{ always() && From 1d9749e81c990c34d5934022022faa79c14aeb4d Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Fri, 2 May 2025 09:17:37 -0400 Subject: [PATCH 16/24] debug --- .github/actions/fetch_ctk/action.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/actions/fetch_ctk/action.yml b/.github/actions/fetch_ctk/action.yml index b076e880e..98ef25f77 100644 --- a/.github/actions/fetch_ctk/action.yml +++ b/.github/actions/fetch_ctk/action.yml @@ -108,7 +108,8 @@ runs: CUDA_PATH="./cuda_toolkit" mkdir -p $CUDA_PATH - rsync -a $CACHE_TMP_DIR $CUDA_PATH + rsync -av $CACHE_TMP_DIR $CUDA_PATH + ls -l $CUDA_PATH rm -rf $CACHE_TMP_DIR - name: Upload CTK cache From c7f99827cb3733f99b3445505fbba3ea8b622ee0 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Fri, 2 May 2025 10:09:51 -0400 Subject: [PATCH 17/24] trailing slash, it's always the trailing slash --- .github/actions/fetch_ctk/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/fetch_ctk/action.yml b/.github/actions/fetch_ctk/action.yml index 98ef25f77..f7615739d 100644 --- a/.github/actions/fetch_ctk/action.yml +++ b/.github/actions/fetch_ctk/action.yml @@ -108,7 +108,7 @@ runs: CUDA_PATH="./cuda_toolkit" mkdir -p $CUDA_PATH - rsync -av $CACHE_TMP_DIR $CUDA_PATH + rsync -av $CACHE_TMP_DIR/ $CUDA_PATH ls -l $CUDA_PATH rm -rf $CACHE_TMP_DIR From b9fabb63bb37d3d5a4b04eb0ab49ca89bc69a963 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Fri, 2 May 2025 14:35:09 +0000 Subject: [PATCH 18/24] for some reason --strip-components=1 does not work... --- .github/actions/fetch_ctk/action.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/actions/fetch_ctk/action.yml b/.github/actions/fetch_ctk/action.yml index f7615739d..0554bba0a 100644 --- a/.github/actions/fetch_ctk/action.yml +++ b/.github/actions/fetch_ctk/action.yml @@ -125,9 +125,12 @@ runs: shell: bash --noprofile --norc -xeuo pipefail {0} run: | ls -l + CACHE_TMP_DIR="./cache_tmp_dir" CUDA_PATH="./cuda_toolkit" mkdir -p $CUDA_PATH - tar -xzvf $CTK_CACHE_FILENAME -C $CUDA_PATH --strip-components=1 + tar -xzvf $CTK_CACHE_FILENAME + rsync -av $CACHE_TMP_DIR/ $CUDA_PATH + rm -rf $CACHE_TMP_DIR $CTK_CACHE_FILENAME ls -l $CUDA_PATH if [ ! -d "$CUDA_PATH/include" ]; then exit 1 From 045199057851e2d0b4f8929f2d21c42346a82b5b Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Fri, 2 May 2025 14:54:58 +0000 Subject: [PATCH 19/24] the test runners do not have rsync --- .github/actions/fetch_ctk/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/fetch_ctk/action.yml b/.github/actions/fetch_ctk/action.yml index 0554bba0a..b77e5feb2 100644 --- a/.github/actions/fetch_ctk/action.yml +++ b/.github/actions/fetch_ctk/action.yml @@ -34,8 +34,8 @@ runs: uses: ./.github/actions/install_unix_deps continue-on-error: false with: - dependencies: "zstd curl xz-utils" - dependent_exes: "zstd curl xz" + dependencies: "zstd curl xz-utils rsync" + dependent_exes: "zstd curl xz rsync" - name: Download CTK cache id: ctk-get-cache From 69b6ee1ddc7dcb53131d3cb755729ab131eaa97a Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Fri, 2 May 2025 15:38:09 +0000 Subject: [PATCH 20/24] restore full matrix --- .github/workflows/build-and-test.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 2e2f4da1c..a00a94028 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -25,14 +25,14 @@ jobs: matrix: host-platform: - linux-64 - # - linux-aarch64 - # - win-64 + - linux-aarch64 + - win-64 python-version: - # - "3.13" + - "3.13" - "3.12" - # - "3.11" - # - "3.10" - # - "3.9" + - "3.11" + - "3.10" + - "3.9" cuda-version: # Note: this is for build-time only. - "12.8.0" @@ -63,11 +63,11 @@ jobs: uses: actions/setup-python@v5 with: python-version: "3.12" - + - name: Set up MSVC if: ${{ startsWith(matrix.host-platform, 'win') }} uses: ilammy/msvc-dev-cmd@v1 - + - name: Set environment variables run: | PYTHON_VERSION_FORMATTED=$(echo '${{ matrix.python-version }}' | tr -d '.') @@ -79,7 +79,7 @@ jobs: PWD=$(pwd) REPO_DIR=$(cygpath -w $PWD) fi - + echo "CUDA_BINDINGS_PARALLEL_LEVEL=$(nproc)" >> $GITHUB_ENV CUDA_CORE_ARTIFACT_BASENAME="cuda-core-python${PYTHON_VERSION_FORMATTED}-${{ matrix.host-platform }}" echo "CUDA_CORE_ARTIFACT_BASENAME=${CUDA_CORE_ARTIFACT_BASENAME}" >> $GITHUB_ENV @@ -215,13 +215,13 @@ jobs: matrix: host-platform: - linux-64 - # - linux-aarch64 + - linux-aarch64 python-version: - # - "3.13" + - "3.13" - "3.12" - # - "3.11" - # - "3.10" - # - "3.9" + - "3.11" + - "3.10" + - "3.9" cuda-version: # Note: this is for test-time only. - "12.8.0" From ca06f3685dda216639a366543815ffc68878de7c Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Fri, 2 May 2025 18:09:28 -0400 Subject: [PATCH 21/24] move --- .github/{actions/fetch_ctk => workflows}/guess_latest.sh | 0 .github/workflows/test-wheel-linux.yml | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename .github/{actions/fetch_ctk => workflows}/guess_latest.sh (100%) diff --git a/.github/actions/fetch_ctk/guess_latest.sh b/.github/workflows/guess_latest.sh similarity index 100% rename from .github/actions/fetch_ctk/guess_latest.sh rename to .github/workflows/guess_latest.sh diff --git a/.github/workflows/test-wheel-linux.yml b/.github/workflows/test-wheel-linux.yml index 34520fb58..2761f8c40 100644 --- a/.github/workflows/test-wheel-linux.yml +++ b/.github/workflows/test-wheel-linux.yml @@ -91,7 +91,7 @@ jobs: # Only local ctk installs have compute-sanitizer; there is not wheel for it if [[ "${{ inputs.python-version }}" == "3.12" && "${{ inputs.cuda-version }}" != "11.8.0" && "${{ inputs.local-ctk }}" == 1 ]]; then SETUP_SANITIZER=1 - echo "LATEST_CUDA_VERSION=$(bash .github/actions/fetch_ctk/guess_latest.sh)" >> $GITHUB_ENV + echo "LATEST_CUDA_VERSION=$(bash .github/workflows/guess_latest.sh)" >> $GITHUB_ENV else SETUP_SANITIZER=0 fi From b2c6de30eb8998bfc3b858c995bab2b153f80b0f Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Fri, 2 May 2025 18:17:50 -0400 Subject: [PATCH 22/24] avoid rsync... --- .github/actions/fetch_ctk/action.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/actions/fetch_ctk/action.yml b/.github/actions/fetch_ctk/action.yml index b77e5feb2..a0ca358dd 100644 --- a/.github/actions/fetch_ctk/action.yml +++ b/.github/actions/fetch_ctk/action.yml @@ -34,8 +34,8 @@ runs: uses: ./.github/actions/install_unix_deps continue-on-error: false with: - dependencies: "zstd curl xz-utils rsync" - dependent_exes: "zstd curl xz rsync" + dependencies: "zstd curl xz-utils" + dependent_exes: "zstd curl xz" - name: Download CTK cache id: ctk-get-cache @@ -106,11 +106,15 @@ runs: # Note: try to escape | and > ... tar -czvf ${CTK_CACHE_FILENAME} ${CACHE_TMP_DIR} + # "Move" files from temp dir to CUDA_PATH CUDA_PATH="./cuda_toolkit" mkdir -p $CUDA_PATH - rsync -av $CACHE_TMP_DIR/ $CUDA_PATH - ls -l $CUDA_PATH + # We could have just done "rsync -av $CACHE_TMP_DIR/ $CUDA_PATH"; however, + # not all runners have rsync pre-installed (or even installable, such as + # Git Bash). We do it in the dumb way. + cp -r $CACHE_TMP_DIR/*/* $CUDA_PATH rm -rf $CACHE_TMP_DIR + ls -l $CUDA_PATH - name: Upload CTK cache if: ${{ always() && @@ -129,7 +133,8 @@ runs: CUDA_PATH="./cuda_toolkit" mkdir -p $CUDA_PATH tar -xzvf $CTK_CACHE_FILENAME - rsync -av $CACHE_TMP_DIR/ $CUDA_PATH + # Can't use rsync here, see above + cp -r $CACHE_TMP_DIR/*/* $CUDA_PATH rm -rf $CACHE_TMP_DIR $CTK_CACHE_FILENAME ls -l $CUDA_PATH if [ ! -d "$CUDA_PATH/include" ]; then From 5376e7902cfff8ff9fecfa42a130bba406662ca4 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Fri, 2 May 2025 18:26:50 -0400 Subject: [PATCH 23/24] fix --- .github/actions/fetch_ctk/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/fetch_ctk/action.yml b/.github/actions/fetch_ctk/action.yml index a0ca358dd..2073530f2 100644 --- a/.github/actions/fetch_ctk/action.yml +++ b/.github/actions/fetch_ctk/action.yml @@ -112,7 +112,7 @@ runs: # We could have just done "rsync -av $CACHE_TMP_DIR/ $CUDA_PATH"; however, # not all runners have rsync pre-installed (or even installable, such as # Git Bash). We do it in the dumb way. - cp -r $CACHE_TMP_DIR/*/* $CUDA_PATH + cp -r $CACHE_TMP_DIR/* $CUDA_PATH rm -rf $CACHE_TMP_DIR ls -l $CUDA_PATH @@ -134,7 +134,7 @@ runs: mkdir -p $CUDA_PATH tar -xzvf $CTK_CACHE_FILENAME # Can't use rsync here, see above - cp -r $CACHE_TMP_DIR/*/* $CUDA_PATH + cp -r $CACHE_TMP_DIR/* $CUDA_PATH rm -rf $CACHE_TMP_DIR $CTK_CACHE_FILENAME ls -l $CUDA_PATH if [ ! -d "$CUDA_PATH/include" ]; then From 718f111b9e8afa2bfeafac4b3c37b3976c22e9ac Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Fri, 2 May 2025 21:44:52 -0400 Subject: [PATCH 24/24] add inline comments --- .github/actions/fetch_ctk/action.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/actions/fetch_ctk/action.yml b/.github/actions/fetch_ctk/action.yml index 2073530f2..403cb423b 100644 --- a/.github/actions/fetch_ctk/action.yml +++ b/.github/actions/fetch_ctk/action.yml @@ -50,6 +50,8 @@ runs: if: ${{ steps.ctk-get-cache.outputs.cache-hit != 'true' }} shell: bash --noprofile --norc -xeuo pipefail {0} run: | + # Everything under this folder is packed and stored in the GitHub Cache space, + # and unpacked after retrieving from the cache. CACHE_TMP_DIR="./cache_tmp_dir" rm -rf $CACHE_TMP_DIR mkdir $CACHE_TMP_DIR @@ -109,7 +111,7 @@ runs: # "Move" files from temp dir to CUDA_PATH CUDA_PATH="./cuda_toolkit" mkdir -p $CUDA_PATH - # We could have just done "rsync -av $CACHE_TMP_DIR/ $CUDA_PATH"; however, + # Unfortunately we cannot use "rsync -av $CACHE_TMP_DIR/ $CUDA_PATH" because # not all runners have rsync pre-installed (or even installable, such as # Git Bash). We do it in the dumb way. cp -r $CACHE_TMP_DIR/* $CUDA_PATH