diff --git a/packaging/build_cmake.sh b/packaging/build_cmake.sh
index e67b7fe5072..eb9e3608063 100755
--- a/packaging/build_cmake.sh
+++ b/packaging/build_cmake.sh
@@ -36,7 +36,13 @@ else
   MKL_CONSTRAINT=''
 fi
 
-conda install -yq \pytorch=$PYTORCH_VERSION $CONDA_CUDATOOLKIT_CONSTRAINT $CONDA_CPUONLY_FEATURE $MKL_CONSTRAINT -c "pytorch-${UPLOAD_CHANNEL}"
+if [[ $CONDA_BUILD_VARIANT == "cpu" ]]; then
+  PYTORCH_MUTEX_CONSTRAINT='pytorch-mutex=1.0=cpu'
+else
+  PYTORCH_MUTEX_CONSTRAINT=''
+fi
+
+conda install -yq \pytorch=$PYTORCH_VERSION $CONDA_CUDATOOLKIT_CONSTRAINT $PYTORCH_MUTEX_CONSTRAINT $MKL_CONSTRAINT -c "pytorch-${UPLOAD_CHANNEL}"
 TORCH_PATH=$(dirname $(python -c "import torch; print(torch.__file__)"))
 
 if [[ "$(uname)" == Darwin || "$OSTYPE" == "msys" ]]; then
diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash
index b0f2bc0580c..897cc103345 100644
--- a/packaging/pkg_helpers.bash
+++ b/packaging/pkg_helpers.bash
@@ -287,9 +287,9 @@ setup_conda_pytorch_constraint() {
 
 # Translate CUDA_VERSION into CUDA_CUDATOOLKIT_CONSTRAINT
 setup_conda_cudatoolkit_constraint() {
-  export CONDA_CPUONLY_FEATURE=""
+  export CONDA_BUILD_VARIANT="cuda"
   if [[ "$(uname)" == Darwin ]]; then
-    export CONDA_CUDATOOLKIT_CONSTRAINT=""
+    export CONDA_BUILD_VARIANT="cpu"
   else
     case "$CU_VERSION" in
       cu113)
@@ -318,7 +318,7 @@ setup_conda_cudatoolkit_constraint() {
         ;;
       cpu)
         export CONDA_CUDATOOLKIT_CONSTRAINT=""
-        export CONDA_CPUONLY_FEATURE="- cpuonly"
+        export CONDA_BUILD_VARIANT="cpu"
         ;;
       *)
         echo "Unrecognized CU_VERSION=$CU_VERSION"
@@ -329,10 +329,10 @@ setup_conda_cudatoolkit_constraint() {
 }
 
 setup_conda_cudatoolkit_plain_constraint() {
-  export CONDA_CPUONLY_FEATURE=""
+  export CONDA_BUILD_VARIANT="cuda"
   export CMAKE_USE_CUDA=1
   if [[ "$(uname)" == Darwin ]]; then
-    export CONDA_CUDATOOLKIT_CONSTRAINT=""
+    export CONDA_BUILD_VARIANT="cpu"
     export CMAKE_USE_CUDA=0
   else
     case "$CU_VERSION" in
@@ -359,7 +359,7 @@ setup_conda_cudatoolkit_plain_constraint() {
         ;;
       cpu)
         export CONDA_CUDATOOLKIT_CONSTRAINT=""
-        export CONDA_CPUONLY_FEATURE="cpuonly"
+        export CONDA_BUILD_VARIANT="cpu"
         export CMAKE_USE_CUDA=0
         ;;
       *)
diff --git a/packaging/torchvision/meta.yaml b/packaging/torchvision/meta.yaml
index e6402a9d4c9..620d95ed516 100644
--- a/packaging/torchvision/meta.yaml
+++ b/packaging/torchvision/meta.yaml
@@ -1,3 +1,4 @@
+{% set build_variant = environ.get('CONDA_BUILD_VARIANT', 'cpu') %}
 package:
   name: torchvision
   version: "{{ environ.get('BUILD_VERSION') }}"
@@ -16,9 +17,9 @@ requirements:
   host:
     - python
     - setuptools
+    - pytorch-mutex 1.0 {{ build_variant }}  # [not osx ]
     {{ environ.get('CONDA_PYTORCH_BUILD_CONSTRAINT') }}
-    {{ environ.get('CONDA_CUDATOOLKIT_CONSTRAINT') }}
-    {{ environ.get('CONDA_CPUONLY_FEATURE') }}
+    {{ environ.get('CONDA_CUDATOOLKIT_CONSTRAINT', '') }}
 
   run:
     - python
@@ -27,8 +28,17 @@ requirements:
     - ffmpeg >=4.2  # [not win]
     - jpeg
     - pillow >=5.3.0
+    - pytorch-mutex 1.0 {{ build_variant }}  # [not osx ]
     {{ environ.get('CONDA_PYTORCH_CONSTRAINT') }}
-    {{ environ.get('CONDA_CUDATOOLKIT_CONSTRAINT') }}
+    {{ environ.get('CONDA_CUDATOOLKIT_CONSTRAINT', '') }}
+
+  {% if build_variant == 'cpu' %}
+  run_constrained:
+    - cpuonly
+  {% elif not osx %}
+  run_constrained:
+     - cpuonly <0
+  {% endif %}
 
 build:
   string: py{{py}}_{{ environ['CU_VERSION'] }}
@@ -38,8 +48,6 @@ build:
     - FORCE_CUDA
     - BUILD_VERSION
     - TORCH_CUDA_ARCH_LIST
-  features:
-    {{ environ.get('CONDA_CPUONLY_FEATURE') }}
 
 test:
   imports: