diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml
index 03e4b2db121..80cbafd2b93 100644
--- a/.github/workflows/test-macos.yml
+++ b/.github/workflows/test-macos.yml
@@ -13,15 +13,13 @@ jobs:
   unittests:
     strategy:
       matrix:
-        python-version:
-          - "3.8"
-          - "3.9"
-          - "3.10"
-          - "3.11"
-        runner: ["macos-12"]
         include:
+          # Test against the most popular version of Python (at the time of commit 40% of torch downloads use 3.8)
           - python-version: "3.8"
-            runner: macos-m1-12
+            runner: "macos-12"
+          # Minimum version available for Apple Silicon is 3.9, so just use that
+          - python-version: "3.9"
+            runner: "macos-m1-12"
       fail-fast: false
     uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
     with: