milestone1: naive_intNwo + eval/benchmark

Author: Hanxian97

torchao/quantization/prototype/mixed_precision/run_layer_sensitive_study.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+GPUS=(0 1 2 3 4 5)
+
+CONFIGS1=("2" "3" "4" "5" "6" "8")
+
+PYTHON_SCRIPT="scripts/sensitivity_study.py"
+
+for LAYER in {0..31}; do
+  for i in "${!GPUS[@]}"; do
+    GPU="${GPUS[$i]}"
+    CONFIG1="${CONFIGS1[$i]}"
+
+    LOG_FILE="Sensi_${LAYER}_${CONFIG1}.txt"
+
+    CUDA_VISIBLE_DEVICES=$GPU python $PYTHON_SCRIPT --repo_id=checkpoints/meta-llama/Meta-Llama-3-8B --quantization=$CONFIG1 --layer=$LAYER &>"$LOG_FILE" &
+  done
+
+  wait
+done
+
+echo "All processes are complete."

torchao/quantization/prototype/mixed_precision/run_mp_quant.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# List of GPUs to use
+GPUS=(0 1 2 3 4 5)
+
+# List of configuration files
+CONFIGS1=(8 8 8 8 8 16)
+CONFIGS2=(6 5 4 3 2 8)
+
+#CONFIGS1=(16 16 16 16 16)
+#CONFIGS2=(6 5 4 3 2)
+
+#CONFIGS1=(5 5 5 6 6 6 6 3)
+#CONFIGS2=(4 3 2 5 4 3 2 2)
+
+PYTHON_SCRIPT="scripts/mp_quant_eval.py"
+
+for i in "${!GPUS[@]}"; do
+  GPU="${GPUS[$i]}"
+  CONFIG1="${CONFIGS1[$i]}"
+  CONFIG2="${CONFIGS2[$i]}"
+
+  LOG_FILE="MP_${CONFIG1}_${CONFIG2}.txt"
+
+  CUDA_VISIBLE_DEVICES=$GPU python $PYTHON_SCRIPT --repo_id=checkpoints/meta-llama/Meta-Llama-3-8B --quantization=MP_llama3 --sensi_bit=$CONFIG1 --non_sensi_bit=$CONFIG2 &>"$LOG_FILE" &
+done
+
+wait
+
+echo "All processes are complete."

torchao/quantization/prototype/mixed_precision/run_mp_quant_benchmark.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+PYTHON_SCRIPT="scripts/generate.py"
+python $PYTHON_SCRIPT --checkpoint_path=checkpoints/meta-llama/Meta-Llama-3-8B/model.pth --compile --quantization=MP_llama3 --sensi_bit=5 --non_sensi_bit=4 --write_result mp_quant_benchmark_results.txt
+python $PYTHON_SCRIPT --checkpoint_path=checkpoints/meta-llama/Meta-Llama-3-8B/model.pth --compile --quantization=MP_llama3 --sensi_bit=4 --non_sensi_bit=3 --write_result mp_quant_benchmark_results.txt
+python $PYTHON_SCRIPT --checkpoint_path=checkpoints/meta-llama/Meta-Llama-3-8B/model.pth --compile --quantization=MP_llama3 --sensi_bit=5 --non_sensi_bit=3 --write_result mp_quant_benchmark_results.txt
+
+
+echo "All processes are complete."

torchao/quantization/prototype/mixed_precision/run_sensi_linear_type.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+CONFIGS1=("q_proj" "k_proj" "v_proj" "o_proj" "gate_proj" "up_proj" "down_proj")
+CONFIGS2=("2" "3" "4" "5" "6" "8")
+
+PYTHON_SCRIPT="scripts/sensitivity_study.py"
+
+GPUS=(0 1 2 3 4 5)
+
+for i in "${!CONFIGS1[@]}"; do
+  CONFIG1="${CONFIGS1[$i]}"
+
+  for j in "${!CONFIGS2[@]}"; do
+    CONFIG2="${CONFIGS2[$j]}"
+    GPU="${GPUS[$j]}"
+
+    LOG_FILE="Sensi_skipsensi_${CONFIG1}_${CONFIG2}.txt"
+
+    CUDA_VISIBLE_DEVICES=$GPU python $PYTHON_SCRIPT --repo_id=checkpoints/meta-llama/Meta-Llama-3-8B --quantization=$CONFIG2 --linear_type=$CONFIG1 &>"$LOG_FILE" &
+  done
+
+  wait
+done
+
+
+echo "All processes are complete."

torchao/quantization/prototype/mixed_precision/run_uni_quant.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# List of GPUs to use
+GPUS=(2 3 4 5 6 7)
+
+# List of configuration files
+#CONFIGS1=("int6wo" "int5wo" "int2wo" "int3wo" "int8wo" "int4wo" "None" "autoquant")
+CONFIGS1=("2" "3" "4" "5" "6" "8")
+#CONFIGS1=(8 8 8 8 8 4 4 16)
+#CONFIGS2=(6 5 4 3 2 3 2 8)
+
+#CONFIGS1=(16 16 16 16 16)
+#CONFIGS2=(6 5 4 3 2)
+
+PYTHON_SCRIPT="scripts/mx_eval.py"
+
+for i in "${!GPUS[@]}"; do
+  GPU="${GPUS[$i]}"
+  CONFIG1="${CONFIGS1[$i]}"
+
+  LOG_FILE="UNI_${CONFIG1}_SYM.txt"
+
+  CUDA_VISIBLE_DEVICES=$GPU python $PYTHON_SCRIPT --repo_id=checkpoints/meta-llama/Meta-Llama-3-8B --quantization=$CONFIG1 --quant_sym=sym &>"$LOG_FILE" &
+done
+
+wait
+
+for i in "${!GPUS[@]}"; do
+  GPU="${GPUS[$i]}"
+  CONFIG1="${CONFIGS1[$i]}"
+
+  LOG_FILE="UNI_${CONFIG1}_ASYM.txt"
+
+  CUDA_VISIBLE_DEVICES=$GPU python $PYTHON_SCRIPT --repo_id=checkpoints/meta-llama/Meta-Llama-3-8B --quantization=$CONFIG1 --quant_sym=asym &>"$LOG_FILE" &
+done
+
+wait
+
+echo "All processes are complete."

torchao/quantization/prototype/mixed_precision/run_uni_quant_benchmark.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+PYTHON_SCRIPT="scripts/generate.py"
+python $PYTHON_SCRIPT --checkpoint_path=checkpoints/meta-llama/Meta-Llama-3-8B/model.pth --compile --quantization=int4wo --write_result uni_quant_benchmark_results.txt
+python $PYTHON_SCRIPT --checkpoint_path=checkpoints/meta-llama/Meta-Llama-3-8B/model.pth --compile --quantization=int8wo --write_result uni_quant_benchmark_results.txt
+python $PYTHON_SCRIPT --checkpoint_path=checkpoints/meta-llama/Meta-Llama-3-8B/model.pth --compile --quantization=2 --write_result uni_quant_benchmark_results.txt
+python $PYTHON_SCRIPT --checkpoint_path=checkpoints/meta-llama/Meta-Llama-3-8B/model.pth --compile --quantization=3 --write_result uni_quant_benchmark_results.txt
+python $PYTHON_SCRIPT --checkpoint_path=checkpoints/meta-llama/Meta-Llama-3-8B/model.pth --compile --quantization=5 --write_result uni_quant_benchmark_results.txt
+python $PYTHON_SCRIPT --checkpoint_path=checkpoints/meta-llama/Meta-Llama-3-8B/model.pth --compile --quantization=6 --write_result uni_quant_benchmark_results.txt
+
+echo "All processes are complete."