From bcdfc54fdc641563c75bfc05fbf703fdb6e7d8e1 Mon Sep 17 00:00:00 2001 From: Jack-Khuu Date: Tue, 12 Nov 2024 11:21:42 -0800 Subject: [PATCH 01/31] Bump PyTorch pin to 20241111 --- install/install_requirements.sh | 4 ++-- torchchat/cli/builder.py | 3 ++- torchchat/distributed/checkpoint.py | 1 + torchchat/export.py | 4 ++-- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/install/install_requirements.sh b/install/install_requirements.sh index 6344509d8..48385f06d 100755 --- a/install/install_requirements.sh +++ b/install/install_requirements.sh @@ -47,10 +47,10 @@ fi # NOTE: If a newly-fetched version of the executorch repo changes the value of # PYTORCH_NIGHTLY_VERSION, you should re-run this script to install the necessary # package versions. -PYTORCH_NIGHTLY_VERSION=dev20241002 +PYTORCH_NIGHTLY_VERSION=dev20241111 # Nightly version for torchvision -VISION_NIGHTLY_VERSION=dev20241002 +VISION_NIGHTLY_VERSION=dev20241111 # Nightly version for torchtune TUNE_NIGHTLY_VERSION=dev20241010 diff --git a/torchchat/cli/builder.py b/torchchat/cli/builder.py index fb2bfb299..84e94701b 100644 --- a/torchchat/cli/builder.py +++ b/torchchat/cli/builder.py @@ -402,6 +402,7 @@ def _load_checkpoint(builder_args: BuilderArgs): os.path.join(builder_args.checkpoint_dir, cp_name), map_location=builder_args.device, mmap=True, + weights_only=False, ) ) checkpoint = {} @@ -706,4 +707,4 @@ def tokenizer_setting_to_name(tiktoken: bool, tokenizers: bool) -> str: return "TikToken" if tokenizers: return "Tokenizers" - return "SentencePiece" \ No newline at end of file + return "SentencePiece" diff --git a/torchchat/distributed/checkpoint.py b/torchchat/distributed/checkpoint.py index 1830e3a75..3fa2d7529 100644 --- a/torchchat/distributed/checkpoint.py +++ b/torchchat/distributed/checkpoint.py @@ -96,6 +96,7 @@ def _load_checkpoints_from_storage( checkpoint_path, map_location=builder_args.device, mmap=True, + weight_only=False, ) diff --git a/torchchat/export.py b/torchchat/export.py index 7c5243b68..9399c2add 100644 --- a/torchchat/export.py +++ b/torchchat/export.py @@ -122,7 +122,7 @@ def export_for_server( from executorch.exir.tracer import Value from torch._export import capture_pre_autograd_graph - from torch.export import export, ExportedProgram + from torch.export import export_for_training, ExportedProgram from torchchat.model import apply_rotary_emb, Attention from torchchat.utils.build_utils import get_precision @@ -238,7 +238,7 @@ def _to_core_aten( raise ValueError( f"Expected passed in model to be an instance of fx.GraphModule, got {type(model)}" ) - core_aten_ep = export(model, example_inputs, dynamic_shapes=dynamic_shapes) + core_aten_ep = export_for_training(model, example_inputs, dynamic_shapes=dynamic_shapes) if verbose: logging.info(f"Core ATen graph:\n{core_aten_ep.graph}") return core_aten_ep From a976734d1474068d07fbc4366090302fb90c6c71 Mon Sep 17 00:00:00 2001 From: Jack-Khuu Date: Tue, 12 Nov 2024 12:02:25 -0800 Subject: [PATCH 02/31] bump to 1112 --- install/install_requirements.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install/install_requirements.sh b/install/install_requirements.sh index 48385f06d..7e7fcc057 100755 --- a/install/install_requirements.sh +++ b/install/install_requirements.sh @@ -47,10 +47,10 @@ fi # NOTE: If a newly-fetched version of the executorch repo changes the value of # PYTORCH_NIGHTLY_VERSION, you should re-run this script to install the necessary # package versions. -PYTORCH_NIGHTLY_VERSION=dev20241111 +PYTORCH_NIGHTLY_VERSION=dev20241112 # Nightly version for torchvision -VISION_NIGHTLY_VERSION=dev20241111 +VISION_NIGHTLY_VERSION=dev20241112 # Nightly version for torchtune TUNE_NIGHTLY_VERSION=dev20241010 From 63289355e69e5e3b062dca18a500853a5ca08e2f Mon Sep 17 00:00:00 2001 From: Jack-Khuu Date: Wed, 13 Nov 2024 11:11:00 -0800 Subject: [PATCH 03/31] Update install_requirements.sh --- install/install_requirements.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/install_requirements.sh b/install/install_requirements.sh index 4b6c0248b..36257b645 100755 --- a/install/install_requirements.sh +++ b/install/install_requirements.sh @@ -60,7 +60,7 @@ echo "Using pip executable: $PIP_EXECUTABLE" # NOTE: If a newly-fetched version of the executorch repo changes the value of # PYTORCH_NIGHTLY_VERSION, you should re-run this script to install the necessary # package versions. -PYTORCH_NIGHTLY_VERSION=dev20241112 +PYTORCH_NIGHTLY_VERSION=dev20241113 # Nightly version for torchvision VISION_NIGHTLY_VERSION=dev20241112 From 7aa96d7f414cd859af672a807090dcd2fabaa08b Mon Sep 17 00:00:00 2001 From: Jack-Khuu Date: Thu, 14 Nov 2024 07:53:43 -0800 Subject: [PATCH 04/31] Update install_requirements.sh --- install/install_requirements.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install/install_requirements.sh b/install/install_requirements.sh index 36257b645..d3e356ea3 100755 --- a/install/install_requirements.sh +++ b/install/install_requirements.sh @@ -60,10 +60,10 @@ echo "Using pip executable: $PIP_EXECUTABLE" # NOTE: If a newly-fetched version of the executorch repo changes the value of # PYTORCH_NIGHTLY_VERSION, you should re-run this script to install the necessary # package versions. -PYTORCH_NIGHTLY_VERSION=dev20241113 +PYTORCH_NIGHTLY_VERSION=dev20241114 # Nightly version for torchvision -VISION_NIGHTLY_VERSION=dev20241112 +VISION_NIGHTLY_VERSION=dev20241114 # Nightly version for torchtune TUNE_NIGHTLY_VERSION=dev20241010 From 774ebb6c2527b087b3f5e28c270b1a17486874ea Mon Sep 17 00:00:00 2001 From: Jack-Khuu Date: Thu, 14 Nov 2024 08:15:15 -0800 Subject: [PATCH 05/31] Update checkpoint.py typo --- torchchat/distributed/checkpoint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchchat/distributed/checkpoint.py b/torchchat/distributed/checkpoint.py index 3fa2d7529..11e397469 100644 --- a/torchchat/distributed/checkpoint.py +++ b/torchchat/distributed/checkpoint.py @@ -96,7 +96,7 @@ def _load_checkpoints_from_storage( checkpoint_path, map_location=builder_args.device, mmap=True, - weight_only=False, + weights_only=False, ) From a6cb90cf8d9df4ccbb0ac3d311085b64b692a377 Mon Sep 17 00:00:00 2001 From: Jack-Khuu Date: Mon, 18 Nov 2024 01:58:09 -0800 Subject: [PATCH 06/31] Update install_requirements.sh --- install/install_requirements.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install/install_requirements.sh b/install/install_requirements.sh index f75572031..a8838c663 100755 --- a/install/install_requirements.sh +++ b/install/install_requirements.sh @@ -62,10 +62,10 @@ echo "Using pip executable: $PIP_EXECUTABLE" # NOTE: If a newly-fetched version of the executorch repo changes the value of # PYTORCH_NIGHTLY_VERSION, you should re-run this script to install the necessary # package versions. -PYTORCH_NIGHTLY_VERSION=dev20241114 +PYTORCH_NIGHTLY_VERSION=dev20241118 # Nightly version for torchvision -VISION_NIGHTLY_VERSION=dev20241114 +VISION_NIGHTLY_VERSION=dev20241118 # Nightly version for torchtune TUNE_NIGHTLY_VERSION=dev20241010 From f9d0a29c7813a5339dae55a13d5a83476703fe94 Mon Sep 17 00:00:00 2001 From: Jack-Khuu Date: Mon, 18 Nov 2024 10:37:05 -0800 Subject: [PATCH 07/31] Update install_requirements.sh --- install/install_requirements.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/install/install_requirements.sh b/install/install_requirements.sh index a8838c663..db27a6a4b 100755 --- a/install/install_requirements.sh +++ b/install/install_requirements.sh @@ -65,7 +65,7 @@ echo "Using pip executable: $PIP_EXECUTABLE" PYTORCH_NIGHTLY_VERSION=dev20241118 # Nightly version for torchvision -VISION_NIGHTLY_VERSION=dev20241118 +VISION_NIGHTLY_VERSION=dev20241114 # Nightly version for torchtune TUNE_NIGHTLY_VERSION=dev20241010 @@ -92,7 +92,6 @@ fi # pip packages needed by exir. REQUIREMENTS_TO_INSTALL=( torch=="2.6.0.${PYTORCH_NIGHTLY_VERSION}" - torchvision=="0.20.0.${VISION_NIGHTLY_VERSION}" torchtune=="0.4.0.${TUNE_NIGHTLY_VERSION}" ) @@ -104,6 +103,13 @@ REQUIREMENTS_TO_INSTALL=( "${REQUIREMENTS_TO_INSTALL[@]}" ) +# Workaround until torchvision nightly is gets bumped +( + set -x + $PIP_EXECUTABLE install --no-deps --extra-index-url "${TORCH_NIGHTLY_URL}" \ + torchvision=="0.20.0.${VISION_NIGHTLY_VERSION}" +) + ( set -x $PIP_EXECUTABLE install torchao=="0.5.0" From c3f18c69582b46b4bae4c42daa0db77157c2c0ed Mon Sep 17 00:00:00 2001 From: Jack-Khuu Date: Tue, 19 Nov 2024 07:47:57 -0800 Subject: [PATCH 08/31] Update install_requirements.sh --- install/install_requirements.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/install_requirements.sh b/install/install_requirements.sh index db27a6a4b..739ea41a1 100755 --- a/install/install_requirements.sh +++ b/install/install_requirements.sh @@ -81,7 +81,7 @@ TUNE_NIGHTLY_VERSION=dev20241010 # with cuda for faster execution on cuda GPUs. if [[ -x "$(command -v nvidia-smi)" ]]; then - TORCH_NIGHTLY_URL="https://download.pytorch.org/whl/nightly/cu121" + TORCH_NIGHTLY_URL="https://download.pytorch.org/whl/nightly/cu124" elif [[ -x "$(command -v rocminfo)" ]]; then TORCH_NIGHTLY_URL="https://download.pytorch.org/whl/nightly/rocm6.2" From 7647d520e36371efe482687079d28caaa731c18f Mon Sep 17 00:00:00 2001 From: Jack-Khuu Date: Mon, 2 Dec 2024 09:01:22 -0800 Subject: [PATCH 09/31] Bump pins, waiting for nvjit fix --- install/.pins/torchao-pin.txt | 2 +- install/install_requirements.sh | 16 ++++++---------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/install/.pins/torchao-pin.txt b/install/.pins/torchao-pin.txt index 40f083249..e5d8cda16 100644 --- a/install/.pins/torchao-pin.txt +++ b/install/.pins/torchao-pin.txt @@ -1 +1 @@ -c8f1174a06dcc0102849c8348ca6573bde8847a9 +a558f7e90012d61cd636bb2068e027cc0062e5ca diff --git a/install/install_requirements.sh b/install/install_requirements.sh index 739ea41a1..417cc25d5 100755 --- a/install/install_requirements.sh +++ b/install/install_requirements.sh @@ -62,10 +62,10 @@ echo "Using pip executable: $PIP_EXECUTABLE" # NOTE: If a newly-fetched version of the executorch repo changes the value of # PYTORCH_NIGHTLY_VERSION, you should re-run this script to install the necessary # package versions. -PYTORCH_NIGHTLY_VERSION=dev20241118 +PYTORCH_NIGHTLY_VERSION=dev20241201 # Nightly version for torchvision -VISION_NIGHTLY_VERSION=dev20241114 +VISION_NIGHTLY_VERSION=dev20241201 # Nightly version for torchtune TUNE_NIGHTLY_VERSION=dev20241010 @@ -92,6 +92,7 @@ fi # pip packages needed by exir. REQUIREMENTS_TO_INSTALL=( torch=="2.6.0.${PYTORCH_NIGHTLY_VERSION}" + torchvision=="0.20.0.${VISION_NIGHTLY_VERSION}" torchtune=="0.4.0.${TUNE_NIGHTLY_VERSION}" ) @@ -103,16 +104,11 @@ REQUIREMENTS_TO_INSTALL=( "${REQUIREMENTS_TO_INSTALL[@]}" ) -# Workaround until torchvision nightly is gets bumped +# For torchao need to install from github since nightly build doesn't have macos build. +# TODO: Remove this and install nightly build, once it supports macos ( set -x - $PIP_EXECUTABLE install --no-deps --extra-index-url "${TORCH_NIGHTLY_URL}" \ - torchvision=="0.20.0.${VISION_NIGHTLY_VERSION}" -) - -( - set -x - $PIP_EXECUTABLE install torchao=="0.5.0" + $PIP_EXECUTABLE install git+https://github.com/pytorch/ao.git@a558f7e90012d61cd636bb2068e027cc0062e5ca ) if [[ -x "$(command -v nvidia-smi)" ]]; then From bb6ca2a9576902a88858c6200013d07b0cb0ac80 Mon Sep 17 00:00:00 2001 From: Jack-Khuu Date: Mon, 2 Dec 2024 09:21:38 -0800 Subject: [PATCH 10/31] Update install_requirements.sh --- install/install_requirements.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install/install_requirements.sh b/install/install_requirements.sh index 417cc25d5..2af5ed6d0 100755 --- a/install/install_requirements.sh +++ b/install/install_requirements.sh @@ -62,10 +62,10 @@ echo "Using pip executable: $PIP_EXECUTABLE" # NOTE: If a newly-fetched version of the executorch repo changes the value of # PYTORCH_NIGHTLY_VERSION, you should re-run this script to install the necessary # package versions. -PYTORCH_NIGHTLY_VERSION=dev20241201 +PYTORCH_NIGHTLY_VERSION=dev20241126 # Nightly version for torchvision -VISION_NIGHTLY_VERSION=dev20241201 +VISION_NIGHTLY_VERSION=dev20241126 # Nightly version for torchtune TUNE_NIGHTLY_VERSION=dev20241010 From eb004671890532de6c2c76ae7035d6122a805d05 Mon Sep 17 00:00:00 2001 From: Jack-Khuu Date: Mon, 2 Dec 2024 09:57:20 -0800 Subject: [PATCH 11/31] bump tune --- install/install_requirements.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/install_requirements.sh b/install/install_requirements.sh index 2af5ed6d0..3e9af1ad7 100755 --- a/install/install_requirements.sh +++ b/install/install_requirements.sh @@ -68,7 +68,7 @@ PYTORCH_NIGHTLY_VERSION=dev20241126 VISION_NIGHTLY_VERSION=dev20241126 # Nightly version for torchtune -TUNE_NIGHTLY_VERSION=dev20241010 +TUNE_NIGHTLY_VERSION=dev20241126 # Uninstall triton, as nightly will depend on pytorch-triton, which is one and the same ( From 673f5aba6820652e012f6c91dc82eb61d2b99fdf Mon Sep 17 00:00:00 2001 From: Jack-Khuu Date: Mon, 2 Dec 2024 10:01:01 -0800 Subject: [PATCH 12/31] fix tune major version --- install/install_requirements.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/install_requirements.sh b/install/install_requirements.sh index 3e9af1ad7..8f0d9ce63 100755 --- a/install/install_requirements.sh +++ b/install/install_requirements.sh @@ -93,7 +93,7 @@ fi REQUIREMENTS_TO_INSTALL=( torch=="2.6.0.${PYTORCH_NIGHTLY_VERSION}" torchvision=="0.20.0.${VISION_NIGHTLY_VERSION}" - torchtune=="0.4.0.${TUNE_NIGHTLY_VERSION}" + torchtune=="0.5.0.${TUNE_NIGHTLY_VERSION}" ) # Install the requirements. --extra-index-url tells pip to look for package From da0a26d2bcca70a737cb9d54c98243ef57cd0eb0 Mon Sep 17 00:00:00 2001 From: Jack-Khuu Date: Mon, 2 Dec 2024 15:06:34 -0800 Subject: [PATCH 13/31] Bump AO pin to pick up import fix --- install/.pins/torchao-pin.txt | 2 +- install/install_requirements.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/install/.pins/torchao-pin.txt b/install/.pins/torchao-pin.txt index e5d8cda16..a46762aad 100644 --- a/install/.pins/torchao-pin.txt +++ b/install/.pins/torchao-pin.txt @@ -1 +1 @@ -a558f7e90012d61cd636bb2068e027cc0062e5ca +2f97b0955953fa1a46594a27f0df2bc48d93e79d diff --git a/install/install_requirements.sh b/install/install_requirements.sh index 8f0d9ce63..02b12756f 100755 --- a/install/install_requirements.sh +++ b/install/install_requirements.sh @@ -108,7 +108,7 @@ REQUIREMENTS_TO_INSTALL=( # TODO: Remove this and install nightly build, once it supports macos ( set -x - $PIP_EXECUTABLE install git+https://github.com/pytorch/ao.git@a558f7e90012d61cd636bb2068e027cc0062e5ca + $PIP_EXECUTABLE install git+https://github.com/pytorch/ao.git@2f97b0955953fa1a46594a27f0df2bc48d93e79d ) if [[ -x "$(command -v nvidia-smi)" ]]; then From 2530e71879f49cd48d887f082cd4d3bc049d45f8 Mon Sep 17 00:00:00 2001 From: Jack-Khuu Date: Mon, 2 Dec 2024 16:44:37 -0800 Subject: [PATCH 14/31] misc --- .github/workflows/pull.yml | 1 + torchchat/cli/builder.py | 2 ++ torchchat/utils/gguf_loader.py | 13 ++++++++++--- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index ee7270a5d..c48436a80 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -731,6 +731,7 @@ jobs: git clone https://github.com/ggerganov/llama.cpp.git pushd llama.cpp + git checkout 64ed2091b24b2f9747148fdf49a34ed5938762c3 make popd diff --git a/torchchat/cli/builder.py b/torchchat/cli/builder.py index 2028050b1..2773c8372 100644 --- a/torchchat/cli/builder.py +++ b/torchchat/cli/builder.py @@ -373,6 +373,8 @@ def _load_model_gguf(builder_args: BuilderArgs) -> Model: kwargs = {} else: kwargs = builder_args.gguf_kwargs + + kwargs.setdefault("device", builder_args.device) model = Model.from_gguf(builder_args.gguf_path, **kwargs) return model diff --git a/torchchat/utils/gguf_loader.py b/torchchat/utils/gguf_loader.py index 309ff807c..206aaec06 100644 --- a/torchchat/utils/gguf_loader.py +++ b/torchchat/utils/gguf_loader.py @@ -570,6 +570,7 @@ def load_model_and_state_dict( load_state_dict: bool = True, load_as_quantized: bool = True, inner_k_tiles=8, + device="cpu", ) -> torch.nn.Module: """ Parses the GGUF file and returns an nn.Module on meta device along with a state_dict @@ -609,9 +610,15 @@ def load_model_and_state_dict( q, s, z = Q4_0.unpack(t) scales_and_zeros = pack_scales_and_zeros(s, z) q_uint8 = (q[::, ::2] << 4 | q[::, 1::2]).to(torch.uint8) - weight_int4pack = torch.ops.aten._convert_weight_to_int4pack( - q_uint8, inner_k_tiles - ) + + if torch.device(device).type == "cpu": + weight_int4pack = torch.ops.aten._convert_weight_to_int4pack_for_cpu( + q_uint8, inner_k_tiles + ) + else: + weight_int4pack = torch.ops.aten._convert_weight_to_int4pack( + q_uint8, inner_k_tiles + ) state_dict[f"{fqn}.weight"] = weight_int4pack state_dict[f"{fqn}.scales_and_zeros"] = scales_and_zeros From 1ada559a7940b7f2bf71bd53c0fe41ca83be935d Mon Sep 17 00:00:00 2001 From: Jack-Khuu Date: Mon, 2 Dec 2024 18:29:40 -0800 Subject: [PATCH 15/31] Update linux_job CI to v2 --- .github/workflows/more-tests.yml | 2 +- .github/workflows/periodic.yml | 2 +- .github/workflows/pull.yml | 12 ++++++------ .github/workflows/run-readme-periodic.yml | 6 +++--- .github/workflows/run-readme-pr.yml | 24 +++++++++++------------ .github/workflows/runner-cuda-dtype.yml | 2 +- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/.github/workflows/more-tests.yml b/.github/workflows/more-tests.yml index 1e0652c96..be9485e9b 100644 --- a/.github/workflows/more-tests.yml +++ b/.github/workflows/more-tests.yml @@ -9,7 +9,7 @@ on: jobs: test-cuda: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main with: runner: linux.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda diff --git a/.github/workflows/periodic.yml b/.github/workflows/periodic.yml index a9561e3e8..5edb7fb3f 100644 --- a/.github/workflows/periodic.yml +++ b/.github/workflows/periodic.yml @@ -108,7 +108,7 @@ jobs: set -eux PYTHONPATH="${PWD}" python .ci/scripts/gather_test_models.py --event "periodic" --backend "gpu" test-gpu: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main name: test-gpu (${{ matrix.platform }}, ${{ matrix.model_name }}) needs: gather-models-gpu secrets: inherit diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index c48436a80..8b19f3b17 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -215,7 +215,7 @@ jobs: set -eux PYTHONPATH="${PWD}" python .ci/scripts/gather_test_models.py --event "pull_request" --backend "gpu" test-gpu-compile: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main name: test-gpu-compile (${{ matrix.platform }}, ${{ matrix.model_name }}) needs: gather-models-gpu strategy: @@ -250,7 +250,7 @@ jobs: echo "::endgroup::" test-gpu-aoti-bfloat16: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main name: test-gpu-aoti-bfloat16 (${{ matrix.platform }}, ${{ matrix.model_name }}) needs: gather-models-gpu strategy: @@ -291,7 +291,7 @@ jobs: echo "::endgroup::" test-gpu-aoti-float32: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main name: test-gpu-aoti-float32 (${{ matrix.platform }}, ${{ matrix.model_name }}) needs: gather-models-gpu strategy: @@ -337,7 +337,7 @@ jobs: echo "::endgroup::" test-gpu-aoti-float16: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main name: test-gpu-aoti-float16 (${{ matrix.platform }}, ${{ matrix.model_name }}) needs: gather-models-gpu strategy: @@ -384,7 +384,7 @@ jobs: echo "::endgroup::" test-gpu-eval-sanity-check: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main name: test-gpu-eval-sanity-check (${{ matrix.platform }}, ${{ matrix.model_name }}) needs: gather-models-gpu strategy: @@ -1031,7 +1031,7 @@ jobs: echo "Tests complete." test-build-runner-et-android: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main with: runner: linux.4xlarge script: | diff --git a/.github/workflows/run-readme-periodic.yml b/.github/workflows/run-readme-periodic.yml index 6a933b5f1..90b57f248 100644 --- a/.github/workflows/run-readme-periodic.yml +++ b/.github/workflows/run-readme-periodic.yml @@ -10,7 +10,7 @@ on: jobs: test-readme: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main secrets: inherit with: runner: linux.g5.4xlarge.nvidia.gpu @@ -44,7 +44,7 @@ jobs: test-quantization-any: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main with: runner: linux.g5.4xlarge.nvidia.gpu secrets: inherit @@ -76,7 +76,7 @@ jobs: echo "::endgroup::" test-gguf-any: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main secrets: inherit with: runner: linux.g5.4xlarge.nvidia.gpu diff --git a/.github/workflows/run-readme-pr.yml b/.github/workflows/run-readme-pr.yml index 4e5e6d014..7bb282679 100644 --- a/.github/workflows/run-readme-pr.yml +++ b/.github/workflows/run-readme-pr.yml @@ -9,7 +9,7 @@ on: jobs: test-readme-any: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main with: runner: linux.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda @@ -33,7 +33,7 @@ jobs: echo "::endgroup::" test-readme-cpu: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main with: runner: linux.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda @@ -57,7 +57,7 @@ jobs: echo "::endgroup::" test-quantization-any: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main with: runner: linux.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda @@ -81,7 +81,7 @@ jobs: echo "::endgroup::" test-quantization-cpu: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main with: runner: linux.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda @@ -100,7 +100,7 @@ jobs: TORCHCHAT_DEVICE=cpu .ci/scripts/run-docs quantization test-gguf-any: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main with: runner: linux.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda @@ -124,7 +124,7 @@ jobs: echo "::endgroup::" test-gguf-cpu: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main with: runner: linux.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda @@ -149,7 +149,7 @@ jobs: test-advanced-any: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main with: runner: linux.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda @@ -174,7 +174,7 @@ jobs: test-advanced-cpu: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main with: runner: linux.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda @@ -198,7 +198,7 @@ jobs: echo "::endgroup::" test-evaluation-any: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main with: runner: linux.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda @@ -222,7 +222,7 @@ jobs: echo "::endgroup::" test-evaluation-cpu: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main with: runner: linux.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda @@ -246,7 +246,7 @@ jobs: echo "::endgroup::" test-multimodal-any: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main with: runner: linux.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda @@ -270,7 +270,7 @@ jobs: echo "::endgroup::" test-multimodal-cpu: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main with: runner: linux.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda diff --git a/.github/workflows/runner-cuda-dtype.yml b/.github/workflows/runner-cuda-dtype.yml index b83b9904b..4786e1654 100644 --- a/.github/workflows/runner-cuda-dtype.yml +++ b/.github/workflows/runner-cuda-dtype.yml @@ -9,7 +9,7 @@ on: jobs: test-runner-aot-cuda: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main with: runner: linux.g5.4xlarge.nvidia.gpu secrets-env: "HF_TOKEN_PERIODIC" From f58c22e49ddede2554b4d63e82b04fcefc589397 Mon Sep 17 00:00:00 2001 From: Jack-Khuu Date: Tue, 3 Dec 2024 17:38:28 -0800 Subject: [PATCH 16/31] Update install_requirements.sh PT pin to 1202 --- install/install_requirements.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install/install_requirements.sh b/install/install_requirements.sh index 02b12756f..f9ed7bf05 100755 --- a/install/install_requirements.sh +++ b/install/install_requirements.sh @@ -62,10 +62,10 @@ echo "Using pip executable: $PIP_EXECUTABLE" # NOTE: If a newly-fetched version of the executorch repo changes the value of # PYTORCH_NIGHTLY_VERSION, you should re-run this script to install the necessary # package versions. -PYTORCH_NIGHTLY_VERSION=dev20241126 +PYTORCH_NIGHTLY_VERSION=dev20241202 # Nightly version for torchvision -VISION_NIGHTLY_VERSION=dev20241126 +VISION_NIGHTLY_VERSION=dev20241202 # Nightly version for torchtune TUNE_NIGHTLY_VERSION=dev20241126 From 2ece601976437ddca858e4943d49eef4481d8cf4 Mon Sep 17 00:00:00 2001 From: Jack-Khuu Date: Wed, 4 Dec 2024 14:36:30 -0800 Subject: [PATCH 17/31] Vision nightly is delayed --- install/install_requirements.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/install/install_requirements.sh b/install/install_requirements.sh index f9ed7bf05..07d40bd9f 100755 --- a/install/install_requirements.sh +++ b/install/install_requirements.sh @@ -65,7 +65,7 @@ echo "Using pip executable: $PIP_EXECUTABLE" PYTORCH_NIGHTLY_VERSION=dev20241202 # Nightly version for torchvision -VISION_NIGHTLY_VERSION=dev20241202 +VISION_NIGHTLY_VERSION=dev20241126 # Nightly version for torchtune TUNE_NIGHTLY_VERSION=dev20241126 @@ -92,7 +92,6 @@ fi # pip packages needed by exir. REQUIREMENTS_TO_INSTALL=( torch=="2.6.0.${PYTORCH_NIGHTLY_VERSION}" - torchvision=="0.20.0.${VISION_NIGHTLY_VERSION}" torchtune=="0.5.0.${TUNE_NIGHTLY_VERSION}" ) @@ -104,6 +103,13 @@ REQUIREMENTS_TO_INSTALL=( "${REQUIREMENTS_TO_INSTALL[@]}" ) +# Workaround until torchvision nightly gets bumped +( + set -x + $PIP_EXECUTABLE install --no-deps --extra-index-url "${TORCH_NIGHTLY_URL}" \ + torchvision=="0.20.0.${VISION_NIGHTLY_VERSION}" +) + # For torchao need to install from github since nightly build doesn't have macos build. # TODO: Remove this and install nightly build, once it supports macos ( From 565338b884442217733c79d598cf04005c1fcff4 Mon Sep 17 00:00:00 2001 From: Jack-Khuu Date: Wed, 4 Dec 2024 17:28:45 -0800 Subject: [PATCH 18/31] Bump Cuda version; drop PT version to one with vision nightly --- .github/workflows/more-tests.yml | 2 +- .github/workflows/periodic.yml | 2 +- .github/workflows/pull.yml | 10 +++++----- .github/workflows/run-readme-periodic.yml | 6 +++--- .github/workflows/run-readme-pr.yml | 24 +++++++++++------------ .github/workflows/runner-cuda-dtype.yml | 2 +- install/install_requirements.sh | 10 ++-------- 7 files changed, 25 insertions(+), 31 deletions(-) diff --git a/.github/workflows/more-tests.yml b/.github/workflows/more-tests.yml index be9485e9b..2a4a14166 100644 --- a/.github/workflows/more-tests.yml +++ b/.github/workflows/more-tests.yml @@ -13,7 +13,7 @@ jobs: with: runner: linux.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda - gpu-arch-version: "12.1" + gpu-arch-version: "12.4" timeout: 60 script: | echo "::group::Print machine info" diff --git a/.github/workflows/periodic.yml b/.github/workflows/periodic.yml index 5edb7fb3f..5a0d9920b 100644 --- a/.github/workflows/periodic.yml +++ b/.github/workflows/periodic.yml @@ -119,7 +119,7 @@ jobs: secrets-env: "HF_TOKEN_PERIODIC" runner: ${{ matrix.runner }} gpu-arch-type: cuda - gpu-arch-version: "12.1" + gpu-arch-version: "12.4" script: | echo "::group::Print machine info" nvidia-smi diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index 8b19f3b17..d1a1c3fca 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -224,7 +224,7 @@ jobs: with: runner: linux.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda - gpu-arch-version: "12.1" + gpu-arch-version: "12.4" script: | echo "::group::Print machine info" nvidia-smi @@ -259,7 +259,7 @@ jobs: with: runner: linux.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda - gpu-arch-version: "12.1" + gpu-arch-version: "12.4" timeout: 60 script: | echo "::group::Print machine info" @@ -300,7 +300,7 @@ jobs: with: runner: linux.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda - gpu-arch-version: "12.1" + gpu-arch-version: "12.4" script: | echo "::group::Print machine info" nvidia-smi @@ -346,7 +346,7 @@ jobs: with: runner: linux.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda - gpu-arch-version: "12.1" + gpu-arch-version: "12.4" script: | echo "::group::Print machine info" nvidia-smi @@ -393,7 +393,7 @@ jobs: with: runner: linux.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda - gpu-arch-version: "12.1" + gpu-arch-version: "12.4" script: | echo "::group::Print machine info" nvidia-smi diff --git a/.github/workflows/run-readme-periodic.yml b/.github/workflows/run-readme-periodic.yml index 90b57f248..387f06210 100644 --- a/.github/workflows/run-readme-periodic.yml +++ b/.github/workflows/run-readme-periodic.yml @@ -16,7 +16,7 @@ jobs: runner: linux.g5.4xlarge.nvidia.gpu secrets-env: "HF_TOKEN_PERIODIC" gpu-arch-type: cuda - gpu-arch-version: "12.1" + gpu-arch-version: "12.4" timeout: 60 script: | echo "::group::Print machine info" @@ -49,7 +49,7 @@ jobs: runner: linux.g5.4xlarge.nvidia.gpu secrets: inherit gpu-arch-type: cuda - gpu-arch-version: "12.1" + gpu-arch-version: "12.4" timeout: 60 script: | echo "::group::Print machine info" @@ -82,7 +82,7 @@ jobs: runner: linux.g5.4xlarge.nvidia.gpu secrets-env: "HF_TOKEN_PERIODIC" gpu-arch-type: cuda - gpu-arch-version: "12.1" + gpu-arch-version: "12.4" timeout: 60 script: | echo "::group::Print machine info" diff --git a/.github/workflows/run-readme-pr.yml b/.github/workflows/run-readme-pr.yml index 7bb282679..8a9e14a12 100644 --- a/.github/workflows/run-readme-pr.yml +++ b/.github/workflows/run-readme-pr.yml @@ -13,7 +13,7 @@ jobs: with: runner: linux.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda - gpu-arch-version: "12.1" + gpu-arch-version: "12.4" timeout: 60 script: | echo "::group::Print machine info" @@ -37,7 +37,7 @@ jobs: with: runner: linux.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda - gpu-arch-version: "12.1" + gpu-arch-version: "12.4" timeout: 60 script: | echo "::group::Print machine info" @@ -61,7 +61,7 @@ jobs: with: runner: linux.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda - gpu-arch-version: "12.1" + gpu-arch-version: "12.4" timeout: 60 script: | echo "::group::Print machine info" @@ -85,7 +85,7 @@ jobs: with: runner: linux.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda - gpu-arch-version: "12.1" + gpu-arch-version: "12.4" timeout: 60 script: | echo "::group::Print machine info" @@ -104,7 +104,7 @@ jobs: with: runner: linux.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda - gpu-arch-version: "12.1" + gpu-arch-version: "12.4" timeout: 60 script: | echo "::group::Print machine info" @@ -128,7 +128,7 @@ jobs: with: runner: linux.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda - gpu-arch-version: "12.1" + gpu-arch-version: "12.4" timeout: 60 script: | echo "::group::Print machine info" @@ -153,7 +153,7 @@ jobs: with: runner: linux.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda - gpu-arch-version: "12.1" + gpu-arch-version: "12.4" timeout: 60 script: | echo "::group::Print machine info" @@ -178,7 +178,7 @@ jobs: with: runner: linux.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda - gpu-arch-version: "12.1" + gpu-arch-version: "12.4" timeout: 60 script: | echo "::group::Print machine info" @@ -202,7 +202,7 @@ jobs: with: runner: linux.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda - gpu-arch-version: "12.1" + gpu-arch-version: "12.4" timeout: 60 script: | echo "::group::Print machine info" @@ -226,7 +226,7 @@ jobs: with: runner: linux.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda - gpu-arch-version: "12.1" + gpu-arch-version: "12.4" timeout: 60 script: | echo "::group::Print machine info" @@ -250,7 +250,7 @@ jobs: with: runner: linux.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda - gpu-arch-version: "12.1" + gpu-arch-version: "12.4" timeout: 60 script: | echo "::group::Print machine info" @@ -274,7 +274,7 @@ jobs: with: runner: linux.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda - gpu-arch-version: "12.1" + gpu-arch-version: "12.4" timeout: 60 script: | echo "::group::Print machine info" diff --git a/.github/workflows/runner-cuda-dtype.yml b/.github/workflows/runner-cuda-dtype.yml index 4786e1654..87f309366 100644 --- a/.github/workflows/runner-cuda-dtype.yml +++ b/.github/workflows/runner-cuda-dtype.yml @@ -14,7 +14,7 @@ jobs: runner: linux.g5.4xlarge.nvidia.gpu secrets-env: "HF_TOKEN_PERIODIC" gpu-arch-type: cuda - gpu-arch-version: "12.1" + gpu-arch-version: "12.4" timeout: 60 script: | echo "::group::Print machine info" diff --git a/install/install_requirements.sh b/install/install_requirements.sh index 07d40bd9f..02b12756f 100755 --- a/install/install_requirements.sh +++ b/install/install_requirements.sh @@ -62,7 +62,7 @@ echo "Using pip executable: $PIP_EXECUTABLE" # NOTE: If a newly-fetched version of the executorch repo changes the value of # PYTORCH_NIGHTLY_VERSION, you should re-run this script to install the necessary # package versions. -PYTORCH_NIGHTLY_VERSION=dev20241202 +PYTORCH_NIGHTLY_VERSION=dev20241126 # Nightly version for torchvision VISION_NIGHTLY_VERSION=dev20241126 @@ -92,6 +92,7 @@ fi # pip packages needed by exir. REQUIREMENTS_TO_INSTALL=( torch=="2.6.0.${PYTORCH_NIGHTLY_VERSION}" + torchvision=="0.20.0.${VISION_NIGHTLY_VERSION}" torchtune=="0.5.0.${TUNE_NIGHTLY_VERSION}" ) @@ -103,13 +104,6 @@ REQUIREMENTS_TO_INSTALL=( "${REQUIREMENTS_TO_INSTALL[@]}" ) -# Workaround until torchvision nightly gets bumped -( - set -x - $PIP_EXECUTABLE install --no-deps --extra-index-url "${TORCH_NIGHTLY_URL}" \ - torchvision=="0.20.0.${VISION_NIGHTLY_VERSION}" -) - # For torchao need to install from github since nightly build doesn't have macos build. # TODO: Remove this and install nightly build, once it supports macos ( From 7088e790fa480aca764a48cbbce371ca8616902b Mon Sep 17 00:00:00 2001 From: Jack-Khuu Date: Thu, 5 Dec 2024 08:54:07 -0800 Subject: [PATCH 19/31] Bump to 1205 vision nightly --- install/install_requirements.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install/install_requirements.sh b/install/install_requirements.sh index 02b12756f..888ee8f60 100755 --- a/install/install_requirements.sh +++ b/install/install_requirements.sh @@ -62,10 +62,10 @@ echo "Using pip executable: $PIP_EXECUTABLE" # NOTE: If a newly-fetched version of the executorch repo changes the value of # PYTORCH_NIGHTLY_VERSION, you should re-run this script to install the necessary # package versions. -PYTORCH_NIGHTLY_VERSION=dev20241126 +PYTORCH_NIGHTLY_VERSION=dev20241205 # Nightly version for torchvision -VISION_NIGHTLY_VERSION=dev20241126 +VISION_NIGHTLY_VERSION=dev20241205 # Nightly version for torchtune TUNE_NIGHTLY_VERSION=dev20241126 From 94aa9a86943a8115f8d3cc8ddf69c4f589c184df Mon Sep 17 00:00:00 2001 From: Jack-Khuu Date: Thu, 5 Dec 2024 09:01:17 -0800 Subject: [PATCH 20/31] Vision nightly 1205 needs 1204 torch(?) --- install/install_requirements.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/install_requirements.sh b/install/install_requirements.sh index 888ee8f60..4c9428db2 100755 --- a/install/install_requirements.sh +++ b/install/install_requirements.sh @@ -62,7 +62,7 @@ echo "Using pip executable: $PIP_EXECUTABLE" # NOTE: If a newly-fetched version of the executorch repo changes the value of # PYTORCH_NIGHTLY_VERSION, you should re-run this script to install the necessary # package versions. -PYTORCH_NIGHTLY_VERSION=dev20241205 +PYTORCH_NIGHTLY_VERSION=dev20241204 # Nightly version for torchvision VISION_NIGHTLY_VERSION=dev20241205 From 6e54cbafeb42804d9426f76a13da004ac7f80502 Mon Sep 17 00:00:00 2001 From: Jack-Khuu Date: Thu, 5 Dec 2024 16:37:48 -0800 Subject: [PATCH 21/31] Drop PT version to 1126 (friendly vision version), update devtoolset to 11 for almalinux --- .github/workflows/more-tests.yml | 4 +- .github/workflows/pull.yml | 16 ++++---- .github/workflows/run-readme-periodic.yml | 12 +++--- .github/workflows/run-readme-pr.yml | 48 +++++++++++------------ .github/workflows/runner-cuda-dtype.yml | 4 +- install/install_requirements.sh | 4 +- 6 files changed, 44 insertions(+), 44 deletions(-) diff --git a/.github/workflows/more-tests.yml b/.github/workflows/more-tests.yml index 2a4a14166..b2e53983a 100644 --- a/.github/workflows/more-tests.yml +++ b/.github/workflows/more-tests.yml @@ -21,8 +21,8 @@ jobs: echo "::endgroup::" echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-10-binutils - export PATH=/opt/rh/devtoolset-10/root/usr/bin/:$PATH + yum install -y devtoolset-11-binutils + export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH echo "::endgroup::" diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index d1a1c3fca..6a67c291e 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -267,8 +267,8 @@ jobs: echo "::endgroup::" echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-10-binutils - export PATH=/opt/rh/devtoolset-10/root/usr/bin/:$PATH + yum install -y devtoolset-11-binutils + export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH echo "::endgroup::" echo "::group::Install required packages" @@ -307,8 +307,8 @@ jobs: echo "::endgroup::" echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-10-binutils - export PATH=/opt/rh/devtoolset-10/root/usr/bin/:$PATH + yum install -y devtoolset-11-binutils + export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH echo "::endgroup::" echo "::group::Install required packages" @@ -353,8 +353,8 @@ jobs: echo "::endgroup::" echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-10-binutils - export PATH=/opt/rh/devtoolset-10/root/usr/bin/:$PATH + yum install -y devtoolset-11-binutils + export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH echo "::endgroup::" echo "::group::Install required packages" @@ -400,8 +400,8 @@ jobs: echo "::endgroup::" echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-10-binutils - export PATH=/opt/rh/devtoolset-10/root/usr/bin/:$PATH + yum install -y devtoolset-11-binutils + export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH echo "::endgroup::" echo "::group::Install required packages" diff --git a/.github/workflows/run-readme-periodic.yml b/.github/workflows/run-readme-periodic.yml index 387f06210..ca7ba6c97 100644 --- a/.github/workflows/run-readme-periodic.yml +++ b/.github/workflows/run-readme-periodic.yml @@ -24,8 +24,8 @@ jobs: echo "::endgroup::" echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-10-binutils - export PATH=/opt/rh/devtoolset-10/root/usr/bin/:$PATH + yum install -y devtoolset-11-binutils + export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH echo "::endgroup::" echo "::group::Create script to run README" @@ -57,8 +57,8 @@ jobs: echo "::endgroup::" echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-10-binutils - export PATH=/opt/rh/devtoolset-10/root/usr/bin/:$PATH + yum install -y devtoolset-11-binutils + export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH echo "::endgroup::" echo "::group::Create script to run quantization" @@ -90,8 +90,8 @@ jobs: echo "::endgroup::" echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-10-binutils - export PATH=/opt/rh/devtoolset-10/root/usr/bin/:$PATH + yum install -y devtoolset-11-binutils + export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH echo "::endgroup::" echo "::group::Create script to run gguf" diff --git a/.github/workflows/run-readme-pr.yml b/.github/workflows/run-readme-pr.yml index 8a9e14a12..dbf700862 100644 --- a/.github/workflows/run-readme-pr.yml +++ b/.github/workflows/run-readme-pr.yml @@ -21,8 +21,8 @@ jobs: echo "::endgroup::" echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-10-binutils - export PATH=/opt/rh/devtoolset-10/root/usr/bin/:$PATH + yum install -y devtoolset-11-binutils + export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH echo "::endgroup::" .ci/scripts/run-docs readme @@ -45,8 +45,8 @@ jobs: echo "::endgroup::" echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-10-binutils - export PATH=/opt/rh/devtoolset-10/root/usr/bin/:$PATH + yum install -y devtoolset-11-binutils + export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH echo "::endgroup::" TORCHCHAT_DEVICE=cpu .ci/scripts/run-docs readme @@ -69,8 +69,8 @@ jobs: echo "::endgroup::" echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-10-binutils - export PATH=/opt/rh/devtoolset-10/root/usr/bin/:$PATH + yum install -y devtoolset-11-binutils + export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH echo "::endgroup::" .ci/scripts/run-docs quantization @@ -93,8 +93,8 @@ jobs: echo "::endgroup::" echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-10-binutils - export PATH=/opt/rh/devtoolset-10/root/usr/bin/:$PATH + yum install -y devtoolset-11-binutils + export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH echo "::endgroup::" TORCHCHAT_DEVICE=cpu .ci/scripts/run-docs quantization @@ -112,8 +112,8 @@ jobs: echo "::endgroup::" echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-10-binutils - export PATH=/opt/rh/devtoolset-10/root/usr/bin/:$PATH + yum install -y devtoolset-11-binutils + export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH echo "::endgroup::" .ci/scripts/run-docs gguf @@ -136,8 +136,8 @@ jobs: echo "::endgroup::" echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-10-binutils - export PATH=/opt/rh/devtoolset-10/root/usr/bin/:$PATH + yum install -y devtoolset-11-binutils + export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH echo "::endgroup::" TORCHCHAT_DEVICE=cpu .ci/scripts/run-docs gguf @@ -161,8 +161,8 @@ jobs: echo "::endgroup::" echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-10-binutils - export PATH=/opt/rh/devtoolset-10/root/usr/bin/:$PATH + yum install -y devtoolset-11-binutils + export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH echo "::endgroup::" .ci/scripts/run-docs advanced @@ -186,8 +186,8 @@ jobs: echo "::endgroup::" echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-10-binutils - export PATH=/opt/rh/devtoolset-10/root/usr/bin/:$PATH + yum install -y devtoolset-11-binutils + export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH echo "::endgroup::" TORCHCHAT_DEVICE=cpu .ci/scripts/run-docs advanced @@ -210,8 +210,8 @@ jobs: echo "::endgroup::" echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-10-binutils - export PATH=/opt/rh/devtoolset-10/root/usr/bin/:$PATH + yum install -y devtoolset-11-binutils + export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH echo "::endgroup::" .ci/scripts/run-docs evaluation @@ -234,8 +234,8 @@ jobs: echo "::endgroup::" echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-10-binutils - export PATH=/opt/rh/devtoolset-10/root/usr/bin/:$PATH + yum install -y devtoolset-11-binutils + export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH echo "::endgroup::" TORCHCHAT_DEVICE=cpu .ci/scripts/run-docs evaluation @@ -258,8 +258,8 @@ jobs: echo "::endgroup::" echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-10-binutils - export PATH=/opt/rh/devtoolset-10/root/usr/bin/:$PATH + yum install -y devtoolset-11-binutils + export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH echo "::endgroup::" .ci/scripts/run-docs multimodal @@ -282,8 +282,8 @@ jobs: echo "::endgroup::" echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-10-binutils - export PATH=/opt/rh/devtoolset-10/root/usr/bin/:$PATH + yum install -y devtoolset-11-binutils + export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH echo "::endgroup::" TORCHCHAT_DEVICE=cpu .ci/scripts/run-docs multimodal diff --git a/.github/workflows/runner-cuda-dtype.yml b/.github/workflows/runner-cuda-dtype.yml index 87f309366..81eea17b4 100644 --- a/.github/workflows/runner-cuda-dtype.yml +++ b/.github/workflows/runner-cuda-dtype.yml @@ -22,8 +22,8 @@ jobs: echo "::endgroup::" echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-10-binutils - export PATH=/opt/rh/devtoolset-10/root/usr/bin/:$PATH + yum install -y devtoolset-11-binutils + export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH echo "::endgroup::" diff --git a/install/install_requirements.sh b/install/install_requirements.sh index 4c9428db2..02b12756f 100755 --- a/install/install_requirements.sh +++ b/install/install_requirements.sh @@ -62,10 +62,10 @@ echo "Using pip executable: $PIP_EXECUTABLE" # NOTE: If a newly-fetched version of the executorch repo changes the value of # PYTORCH_NIGHTLY_VERSION, you should re-run this script to install the necessary # package versions. -PYTORCH_NIGHTLY_VERSION=dev20241204 +PYTORCH_NIGHTLY_VERSION=dev20241126 # Nightly version for torchvision -VISION_NIGHTLY_VERSION=dev20241205 +VISION_NIGHTLY_VERSION=dev20241126 # Nightly version for torchtune TUNE_NIGHTLY_VERSION=dev20241126 From a05683ddffd8f1f2d75762f42ca8d07150807674 Mon Sep 17 00:00:00 2001 From: Jack-Khuu Date: Thu, 5 Dec 2024 18:02:57 -0800 Subject: [PATCH 22/31] Test download toolchain instead of binutils --- .github/workflows/more-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/more-tests.yml b/.github/workflows/more-tests.yml index b2e53983a..9dc7d6cbd 100644 --- a/.github/workflows/more-tests.yml +++ b/.github/workflows/more-tests.yml @@ -21,7 +21,7 @@ jobs: echo "::endgroup::" echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-11-binutils + yum install -y devtoolset-11-toolchain export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH echo "::endgroup::" From 411cf94281f22c8369aba01b970c053335b9c7fe Mon Sep 17 00:00:00 2001 From: Jack-Khuu Date: Thu, 5 Dec 2024 19:17:31 -0800 Subject: [PATCH 23/31] Test removing devtoolset --- .github/workflows/more-tests.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/more-tests.yml b/.github/workflows/more-tests.yml index 9dc7d6cbd..f47740fe3 100644 --- a/.github/workflows/more-tests.yml +++ b/.github/workflows/more-tests.yml @@ -20,12 +20,6 @@ jobs: uname -a echo "::endgroup::" - echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-11-toolchain - export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH - echo "::endgroup::" - - echo "::group::Download checkpoints" # Install requirements ./install/install_requirements.sh cuda From 953a42e2f156a516e2ba715d0a190f3af7bbbbbb Mon Sep 17 00:00:00 2001 From: Jack-Khuu Date: Fri, 6 Dec 2024 10:02:19 -0800 Subject: [PATCH 24/31] Remove dep on devtoolset 11 that doesnt' exist on the new machine --- .github/workflows/pull.yml | 20 -------- .github/workflows/run-readme-periodic.yml | 15 ------ .github/workflows/run-readme-pr.yml | 60 ----------------------- .github/workflows/runner-cuda-dtype.yml | 6 --- 4 files changed, 101 deletions(-) diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index 6a67c291e..aa3b2cb9b 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -266,11 +266,6 @@ jobs: nvidia-smi echo "::endgroup::" - echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-11-binutils - export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH - echo "::endgroup::" - echo "::group::Install required packages" ./install/install_requirements.sh cuda pip3 list @@ -306,11 +301,6 @@ jobs: nvidia-smi echo "::endgroup::" - echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-11-binutils - export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH - echo "::endgroup::" - echo "::group::Install required packages" ./install/install_requirements.sh cuda pip list @@ -352,11 +342,6 @@ jobs: nvidia-smi echo "::endgroup::" - echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-11-binutils - export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH - echo "::endgroup::" - echo "::group::Install required packages" ./install/install_requirements.sh cuda pip list @@ -399,11 +384,6 @@ jobs: nvidia-smi echo "::endgroup::" - echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-11-binutils - export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH - echo "::endgroup::" - echo "::group::Install required packages" ./install/install_requirements.sh cuda pip3 list diff --git a/.github/workflows/run-readme-periodic.yml b/.github/workflows/run-readme-periodic.yml index ca7ba6c97..61501e0c4 100644 --- a/.github/workflows/run-readme-periodic.yml +++ b/.github/workflows/run-readme-periodic.yml @@ -23,11 +23,6 @@ jobs: uname -a echo "::endgroup::" - echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-11-binutils - export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH - echo "::endgroup::" - echo "::group::Create script to run README" python3 torchchat/utils/scripts/updown.py --create-sections --file README.md > ./run-readme.sh # for good measure, if something happened to updown processor, @@ -56,11 +51,6 @@ jobs: uname -a echo "::endgroup::" - echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-11-binutils - export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH - echo "::endgroup::" - echo "::group::Create script to run quantization" python3 torchchat/utils/scripts/updown.py --create-sections --file docs/quantization.md > ./run-quantization.sh # for good measure, if something happened to updown processor, @@ -89,11 +79,6 @@ jobs: uname -a echo "::endgroup::" - echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-11-binutils - export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH - echo "::endgroup::" - echo "::group::Create script to run gguf" python3 torchchat/utils/scripts/updown.py --file docs/GGUF.md > ./run-gguf.sh # for good measure, if something happened to updown processor, diff --git a/.github/workflows/run-readme-pr.yml b/.github/workflows/run-readme-pr.yml index dbf700862..578fa217d 100644 --- a/.github/workflows/run-readme-pr.yml +++ b/.github/workflows/run-readme-pr.yml @@ -20,11 +20,6 @@ jobs: uname -a echo "::endgroup::" - echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-11-binutils - export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH - echo "::endgroup::" - .ci/scripts/run-docs readme echo "::group::Completion" @@ -44,11 +39,6 @@ jobs: uname -a echo "::endgroup::" - echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-11-binutils - export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH - echo "::endgroup::" - TORCHCHAT_DEVICE=cpu .ci/scripts/run-docs readme echo "::group::Completion" @@ -68,11 +58,6 @@ jobs: uname -a echo "::endgroup::" - echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-11-binutils - export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH - echo "::endgroup::" - .ci/scripts/run-docs quantization echo "::group::Completion" @@ -92,11 +77,6 @@ jobs: uname -a echo "::endgroup::" - echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-11-binutils - export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH - echo "::endgroup::" - TORCHCHAT_DEVICE=cpu .ci/scripts/run-docs quantization test-gguf-any: @@ -111,11 +91,6 @@ jobs: uname -a echo "::endgroup::" - echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-11-binutils - export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH - echo "::endgroup::" - .ci/scripts/run-docs gguf echo "::group::Completion" @@ -135,11 +110,6 @@ jobs: uname -a echo "::endgroup::" - echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-11-binutils - export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH - echo "::endgroup::" - TORCHCHAT_DEVICE=cpu .ci/scripts/run-docs gguf echo "::group::Completion" @@ -160,11 +130,6 @@ jobs: uname -a echo "::endgroup::" - echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-11-binutils - export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH - echo "::endgroup::" - .ci/scripts/run-docs advanced echo "::group::Completion" @@ -185,11 +150,6 @@ jobs: uname -a echo "::endgroup::" - echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-11-binutils - export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH - echo "::endgroup::" - TORCHCHAT_DEVICE=cpu .ci/scripts/run-docs advanced echo "::group::Completion" @@ -209,11 +169,6 @@ jobs: uname -a echo "::endgroup::" - echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-11-binutils - export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH - echo "::endgroup::" - .ci/scripts/run-docs evaluation echo "::group::Completion" @@ -233,11 +188,6 @@ jobs: uname -a echo "::endgroup::" - echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-11-binutils - export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH - echo "::endgroup::" - TORCHCHAT_DEVICE=cpu .ci/scripts/run-docs evaluation echo "::group::Completion" @@ -257,11 +207,6 @@ jobs: uname -a echo "::endgroup::" - echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-11-binutils - export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH - echo "::endgroup::" - .ci/scripts/run-docs multimodal echo "::group::Completion" @@ -281,9 +226,4 @@ jobs: uname -a echo "::endgroup::" - echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-11-binutils - export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH - echo "::endgroup::" - TORCHCHAT_DEVICE=cpu .ci/scripts/run-docs multimodal diff --git a/.github/workflows/runner-cuda-dtype.yml b/.github/workflows/runner-cuda-dtype.yml index 81eea17b4..1813f483e 100644 --- a/.github/workflows/runner-cuda-dtype.yml +++ b/.github/workflows/runner-cuda-dtype.yml @@ -21,12 +21,6 @@ jobs: uname -a echo "::endgroup::" - echo "::group::Install newer objcopy that supports --set-section-alignment" - yum install -y devtoolset-11-binutils - export PATH=/opt/rh/devtoolset-11/root/usr/bin/:$PATH - echo "::endgroup::" - - echo "::group::Download checkpoints" # Install requirements From 6e8bfb181a4afe11a919f29aac98eb3ae94ef424 Mon Sep 17 00:00:00 2001 From: Jack-Khuu Date: Fri, 6 Dec 2024 10:36:08 -0800 Subject: [PATCH 25/31] Bump ET pin --- install/.pins/et-pin.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/.pins/et-pin.txt b/install/.pins/et-pin.txt index e61fae3a5..bb70ed39d 100644 --- a/install/.pins/et-pin.txt +++ b/install/.pins/et-pin.txt @@ -1 +1 @@ -72b3bb3194c611f7c4861e6f3b24af5de868af72 +98e4dd524f2cb08414ee015b27616229cabc06ba From 59e00d50795aab67bdd2a67c1b2e5896a26f42ae Mon Sep 17 00:00:00 2001 From: Jack-Khuu Date: Fri, 6 Dec 2024 15:42:17 -0800 Subject: [PATCH 26/31] Test nightly with updated vision --- install/install_requirements.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install/install_requirements.sh b/install/install_requirements.sh index 02b12756f..5f32704a9 100755 --- a/install/install_requirements.sh +++ b/install/install_requirements.sh @@ -62,10 +62,10 @@ echo "Using pip executable: $PIP_EXECUTABLE" # NOTE: If a newly-fetched version of the executorch repo changes the value of # PYTORCH_NIGHTLY_VERSION, you should re-run this script to install the necessary # package versions. -PYTORCH_NIGHTLY_VERSION=dev20241126 +PYTORCH_NIGHTLY_VERSION=dev20241206 # Nightly version for torchvision -VISION_NIGHTLY_VERSION=dev20241126 +VISION_NIGHTLY_VERSION=dev20241206 # Nightly version for torchtune TUNE_NIGHTLY_VERSION=dev20241126 From aae4eb38c6c3897bab0ea55c3f6e230c7b44c40e Mon Sep 17 00:00:00 2001 From: Jack-Khuu Date: Fri, 6 Dec 2024 17:26:20 -0800 Subject: [PATCH 27/31] Attempt to account for int4wo packing pt#139611 --- torchchat/utils/gguf_loader.py | 41 ++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/torchchat/utils/gguf_loader.py b/torchchat/utils/gguf_loader.py index 206aaec06..8ef874829 100644 --- a/torchchat/utils/gguf_loader.py +++ b/torchchat/utils/gguf_loader.py @@ -122,7 +122,21 @@ def linear_int4(input, weight_int4pack, scales_and_zeros, out_features, groupsiz input.dtype ) # cast back to input.dtype else: - c = torch.ops.aten._weight_int4pack_mm( + # copied from https://github.com/pytorch/pytorch/blob/0052943bee624c06d8c36a371efdf7b56972ad9e/torch/_meta_registrations.py#L3308 + def meta__weight_int4pack_mm(x, w, q_group_size, q_scale_and_zeros): + torch._check(x.dim() == 2, lambda: "x must be a 2D tensor") + torch._check(w.dim() == 4, lambda: "w must be a 4D tensor") + torch._check( + x.dtype in [torch.float32, torch.float16, torch.bfloat16], + lambda: f"expected x to be f32/f16/bf16, got {x.dtype}", + ) + torch._check( + w.dtype is torch.int32, + lambda: f"expected w to be int32, got {w.dtype}", + ) + return x.new_empty(x.size(0), w.size(0) * 8, dtype=x.dtype) + + c = meta__weight_int4pack_mm( input, weight_int4pack, groupsize, @@ -610,10 +624,29 @@ def load_model_and_state_dict( q, s, z = Q4_0.unpack(t) scales_and_zeros = pack_scales_and_zeros(s, z) q_uint8 = (q[::, ::2] << 4 | q[::, 1::2]).to(torch.uint8) - + if torch.device(device).type == "cpu": - weight_int4pack = torch.ops.aten._convert_weight_to_int4pack_for_cpu( - q_uint8, inner_k_tiles + # Copied from https://github.com/pytorch/pytorch/blob/0052943bee624c06d8c36a371efdf7b56972ad9e/torch/_meta_registrations.py#L3273 + def meta__convert_weight_to_int4pack(w, inner_k_tiles): + torch._check(w.dim() == 2, lambda: "w must be a 2D tensor") + torch._check( + w.dtype is torch.uint8, + lambda: f"expected w to be uint8, got {w.dtype}", + ) + n = w.size(0) + k = w.size(1) * 2 # w is [n][k / 2] uint8 + return w.new_empty( + ( + n // 8, + k // (inner_k_tiles * 16), + 32, + inner_k_tiles // 2, + ), + dtype=torch.int32, + ) + + weight_int4pack = meta__convert_weight_to_int4pack( + q_uint8, inner_k_tiles ) else: weight_int4pack = torch.ops.aten._convert_weight_to_int4pack( From 25da485a9535fca021757f532e2058484fe6b6ee Mon Sep 17 00:00:00 2001 From: Jack-Khuu Date: Fri, 6 Dec 2024 18:32:53 -0800 Subject: [PATCH 28/31] Naive gguf int4wo attempt --- torchchat/utils/gguf_loader.py | 39 ++++------------------------------ 1 file changed, 4 insertions(+), 35 deletions(-) diff --git a/torchchat/utils/gguf_loader.py b/torchchat/utils/gguf_loader.py index 8ef874829..c69bdf469 100644 --- a/torchchat/utils/gguf_loader.py +++ b/torchchat/utils/gguf_loader.py @@ -122,21 +122,7 @@ def linear_int4(input, weight_int4pack, scales_and_zeros, out_features, groupsiz input.dtype ) # cast back to input.dtype else: - # copied from https://github.com/pytorch/pytorch/blob/0052943bee624c06d8c36a371efdf7b56972ad9e/torch/_meta_registrations.py#L3308 - def meta__weight_int4pack_mm(x, w, q_group_size, q_scale_and_zeros): - torch._check(x.dim() == 2, lambda: "x must be a 2D tensor") - torch._check(w.dim() == 4, lambda: "w must be a 4D tensor") - torch._check( - x.dtype in [torch.float32, torch.float16, torch.bfloat16], - lambda: f"expected x to be f32/f16/bf16, got {x.dtype}", - ) - torch._check( - w.dtype is torch.int32, - lambda: f"expected w to be int32, got {w.dtype}", - ) - return x.new_empty(x.size(0), w.size(0) * 8, dtype=x.dtype) - - c = meta__weight_int4pack_mm( + c = torch.ops.aten._weight_int4pack_mm_for_cpu( input, weight_int4pack, groupsize, @@ -626,27 +612,10 @@ def load_model_and_state_dict( q_uint8 = (q[::, ::2] << 4 | q[::, 1::2]).to(torch.uint8) if torch.device(device).type == "cpu": - # Copied from https://github.com/pytorch/pytorch/blob/0052943bee624c06d8c36a371efdf7b56972ad9e/torch/_meta_registrations.py#L3273 - def meta__convert_weight_to_int4pack(w, inner_k_tiles): - torch._check(w.dim() == 2, lambda: "w must be a 2D tensor") - torch._check( - w.dtype is torch.uint8, - lambda: f"expected w to be uint8, got {w.dtype}", + weight_int4pack = ( + torch.ops.aten._convert_weight_to_int4pack_for_cpu( + q, inner_k_tiles ) - n = w.size(0) - k = w.size(1) * 2 # w is [n][k / 2] uint8 - return w.new_empty( - ( - n // 8, - k // (inner_k_tiles * 16), - 32, - inner_k_tiles // 2, - ), - dtype=torch.int32, - ) - - weight_int4pack = meta__convert_weight_to_int4pack( - q_uint8, inner_k_tiles ) else: weight_int4pack = torch.ops.aten._convert_weight_to_int4pack( From a9fa27ee2743d9d82a51d5a0d711a015bef1a6d3 Mon Sep 17 00:00:00 2001 From: Jack-Khuu Date: Tue, 10 Dec 2024 08:21:34 -0800 Subject: [PATCH 29/31] Update install_requirements.sh to 1210 --- install/install_requirements.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install/install_requirements.sh b/install/install_requirements.sh index 5f32704a9..17ae3e2fa 100755 --- a/install/install_requirements.sh +++ b/install/install_requirements.sh @@ -62,10 +62,10 @@ echo "Using pip executable: $PIP_EXECUTABLE" # NOTE: If a newly-fetched version of the executorch repo changes the value of # PYTORCH_NIGHTLY_VERSION, you should re-run this script to install the necessary # package versions. -PYTORCH_NIGHTLY_VERSION=dev20241206 +PYTORCH_NIGHTLY_VERSION=dev20241210 # Nightly version for torchvision -VISION_NIGHTLY_VERSION=dev20241206 +VISION_NIGHTLY_VERSION=dev20241210 # Nightly version for torchtune TUNE_NIGHTLY_VERSION=dev20241126 From bfe58269120b098ee05d5896ff203b1dd4e0da83 Mon Sep 17 00:00:00 2001 From: Jack-Khuu Date: Fri, 13 Dec 2024 11:30:11 -0800 Subject: [PATCH 30/31] Update install_requirements.sh to 20241213 Should fix the MacOS wheel regression --- install/install_requirements.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install/install_requirements.sh b/install/install_requirements.sh index 17ae3e2fa..c406553ea 100755 --- a/install/install_requirements.sh +++ b/install/install_requirements.sh @@ -62,10 +62,10 @@ echo "Using pip executable: $PIP_EXECUTABLE" # NOTE: If a newly-fetched version of the executorch repo changes the value of # PYTORCH_NIGHTLY_VERSION, you should re-run this script to install the necessary # package versions. -PYTORCH_NIGHTLY_VERSION=dev20241210 +PYTORCH_NIGHTLY_VERSION=dev20241213 # Nightly version for torchvision -VISION_NIGHTLY_VERSION=dev20241210 +VISION_NIGHTLY_VERSION=dev20241213 # Nightly version for torchtune TUNE_NIGHTLY_VERSION=dev20241126 From dbb090f1d3c0b53fd1e2865551641be1b8eb1823 Mon Sep 17 00:00:00 2001 From: Jack-Khuu Date: Fri, 13 Dec 2024 11:48:19 -0800 Subject: [PATCH 31/31] Update torchvision minor version to 22 --- install/install_requirements.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/install_requirements.sh b/install/install_requirements.sh index c406553ea..3db559dbc 100755 --- a/install/install_requirements.sh +++ b/install/install_requirements.sh @@ -92,7 +92,7 @@ fi # pip packages needed by exir. REQUIREMENTS_TO_INSTALL=( torch=="2.6.0.${PYTORCH_NIGHTLY_VERSION}" - torchvision=="0.20.0.${VISION_NIGHTLY_VERSION}" + torchvision=="0.22.0.${VISION_NIGHTLY_VERSION}" torchtune=="0.5.0.${TUNE_NIGHTLY_VERSION}" )