Update xfails for torchvision models. (huggingface#310)

monorimet · web-flow · commit d9462877231e · 2022-09-01T13:06:12.000-05:00
diff --git a/tank/alexnet_torch/alexnet_torch_test.py b/tank/alexnet_torch/alexnet_torch_test.py
@@ -70,6 +70,11 @@ def test_module(self, dynamic, device):
                     pytest.xfail(
                         reason="Assert Error:https://github.com/iree-org/iree/issues/10075"
                     )
+            if dynamic == True:
+                pytest.xfail(
+                    reason="https://github.com/nod-ai/SHARK/issues/309"
+                )
+
         self.module_tester.create_and_check_module(dynamic, device)
 
 
diff --git a/tank/facebook_convnext-tiny-224_tf/facebook_convnext-tiny-224_tf_test.py b/tank/facebook_convnext-tiny-224_tf/facebook_convnext-tiny-224_tf_test.py
@@ -49,6 +49,10 @@ def configure(self, pytestconfig):
 
     @parameterized.expand(param_list, name_func=shark_test_name_func)
     def test_module(self, dynamic, device):
+
+        if device in ["gpu", "cuda"]:
+            pytest.xfail(reason="https://github.com/nod-ai/SHARK/issues/311")
+
         self.module_tester.create_and_check_module(dynamic, device)
 
 
diff --git a/tank/facebook_deit-small-distilled-patch16-224_torch/facebook_deit-small-distilled-patch16-224_torch_test.py b/tank/facebook_deit-small-distilled-patch16-224_torch/facebook_deit-small-distilled-patch16-224_torch_test.py
@@ -48,6 +48,9 @@ def test_module(self, dynamic, device):
             pytest.skip(
                 reason="Dynamic Test not Supported: mlir file not found"
             )
+        if device in ["gpu", "cuda"]:
+            pytest.xfail(reason="https://github.com/nod-ai/SHARK/issues/311")
+
         self.module_tester.create_and_check_module(dynamic, device)
 
 
diff --git a/tank/google_vit-base-patch16-224_tf/google_vit-base-patch16-224_tf_test.py b/tank/google_vit-base-patch16-224_tf/google_vit-base-patch16-224_tf_test.py
@@ -46,6 +46,10 @@ def configure(self, pytestconfig):
 
     @parameterized.expand(param_list, name_func=shark_test_name_func)
     def test_module(self, dynamic, device):
+
+        if device in ["gpu", "cuda"]:
+            pytest.xfail(reason="https://github.com/nod-ai/SHARK/issues/311")
+
         self.module_tester.create_and_check_module(dynamic, device)
 
 
diff --git a/tank/google_vit-base-patch16-224_torch/google_vit-base-patch16-224_torch_test.py b/tank/google_vit-base-patch16-224_torch/google_vit-base-patch16-224_torch_test.py
@@ -48,6 +48,10 @@ def test_module(self, dynamic, device):
             pytest.skip(
                 reason="Dynamic tests not supported. mlir file not found."
             )
+
+        if device in ["gpu", "cuda"]:
+            pytest.xfail(reason="https://github.com/nod-ai/SHARK/issues/311")
+
         self.module_tester.create_and_check_module(dynamic, device)
 
 
diff --git a/tank/mobilenet_v3_small_torch/mobilenet_v3_small_torch_test.py b/tank/mobilenet_v3_small_torch/mobilenet_v3_small_torch_test.py
@@ -67,6 +67,11 @@ def test_module(self, dynamic, device):
         elif device in ["vulkan", "metal"]:
             if dynamic == False:
                 pytest.xfail(reason="stuck in the pipeline.")
+            if dynamic == True:
+                pytest.xfail(
+                    reason="https://github.com/nod-ai/SHARK/issues/309"
+                )
+
         self.module_tester.create_and_check_module(dynamic, device)
 
 
diff --git a/tank/nvidia_mit-b0_torch/nvidia_mit-b0_torch_test.py b/tank/nvidia_mit-b0_torch/nvidia_mit-b0_torch_test.py
@@ -48,6 +48,10 @@ def test_module(self, dynamic, device):
             pytest.skip(
                 reason="Dynamic tests not supported. mlir file not found."
             )
+
+        if device in ["gpu", "cuda"]:
+            pytest.xfail(reason="https://github.com/nod-ai/SHARK/issues/311")
+
         self.module_tester.create_and_check_module(dynamic, device)
 
 
diff --git a/tank/resnet101_torch/resnet101_torch_test.py b/tank/resnet101_torch/resnet101_torch_test.py
@@ -65,6 +65,12 @@ def configure(self, pytestconfig):
 
     @parameterized.expand(param_list, name_func=shark_test_name_func)
     def test_module(self, dynamic, device):
+        if device in ["metal", "vulkan"]:
+            if dynamic == True:
+                pytest.xfail(
+                    reason="https://github.com/nod-ai/SHARK/issues/309"
+                )
+
         self.module_tester.create_and_check_module(dynamic, device)
 
 
diff --git a/tank/resnet18_torch/resnet18_torch_test.py b/tank/resnet18_torch/resnet18_torch_test.py
@@ -64,6 +64,13 @@ def configure(self, pytestconfig):
 
     @parameterized.expand(param_list, name_func=shark_test_name_func)
     def test_module(self, dynamic, device):
+
+        if device in ["metal", "vulkan"]:
+            if dynamic == True:
+                pytest.xfail(
+                    reason="https://github.com/nod-ai/SHARK/issues/309"
+                )
+
         self.module_tester.create_and_check_module(dynamic, device)
 
 
diff --git a/tank/resnet50_torch/resnet50_torch_test.py b/tank/resnet50_torch/resnet50_torch_test.py
@@ -69,6 +69,11 @@ def test_module(self, dynamic, device):
         if device in ["metal", "vulkan"]:
             if "m1-moltenvk-macos" in get_vulkan_triple_flag():
                 pytest.xfail(reason="M1: CompilerToolError | M2: Pass")
+            if dynamic == True:
+                pytest.xfail(
+                    reason="https://github.com/nod-ai/SHARK/issues/309"
+                )
+
         self.module_tester.create_and_check_module(dynamic, device)
 
 
diff --git a/tank/squeezenet1_0_torch/squeezenet1_0_torch_test.py b/tank/squeezenet1_0_torch/squeezenet1_0_torch_test.py
@@ -23,16 +23,6 @@ def create_and_check_module(self, dynamic, device):
             "squeezenet1_0", dynamic
         )
 
-        # from shark.shark_importer import SharkImporter
-        # mlir_importer = SharkImporter(
-        #    model,
-        #    (input,),
-        #    frontend="torch",
-        # )
-        # minilm_mlir, func_name = mlir_importer.import_mlir(
-        #    is_dynamic=dynamic, tracing_required=True
-        # )
-
         shark_module = SharkInference(
             model_mlir,
             func_name,
@@ -58,8 +48,6 @@ class SqueezenetModuleTest(unittest.TestCase):
     @pytest.fixture(autouse=True)
     def configure(self, pytestconfig):
         self.module_tester = SqueezenetModuleTester(self)
-        self.module_tester.save_mlir = pytestconfig.getoption("save_mlir")
-        self.module_tester.save_vmfb = pytestconfig.getoption("save_vmfb")
         self.module_tester.benchmark = pytestconfig.getoption("benchmark")
 
     param_list = get_valid_test_params()
@@ -68,6 +56,9 @@ def configure(self, pytestconfig):
     def test_module(self, dynamic, device):
         if device in ["metal", "vulkan"]:
             if dynamic == True:
+                pytest.xfail(
+                    reason="https://github.com/nod-ai/SHARK/issues/309"
+                )
                 if "m1-moltenvk-macos" in get_vulkan_triple_flag():
                     pytest.xfail(
                         reason="https://github.com/iree-org/iree/issues/9972"
diff --git a/tank/wide_resnet50_2_torch/wide_resnet50_2_torch_test.py b/tank/wide_resnet50_2_torch/wide_resnet50_2_torch_test.py
@@ -65,6 +65,13 @@ def configure(self, pytestconfig):
 
     @parameterized.expand(param_list, name_func=shark_test_name_func)
     def test_module(self, dynamic, device):
+
+        if device in ["metal", "vulkan"]:
+            if dynamic == True:
+                pytest.xfail(
+                    reason="https://github.com/nod-ai/SHARK/issues/309"
+                )
+
         self.module_tester.create_and_check_module(dynamic, device)
 
 

Original file line number	Diff line number	Diff line change
`@@ -48,6 +48,9 @@ def test_module(self, dynamic, device):`
`48`	`48`	`pytest.skip(`
`49`	`49`	`reason="Dynamic Test not Supported: mlir file not found"`
`50`	`50`	`)`
	`51`	`+ if device in ["gpu", "cuda"]:`
	`52`	`+ pytest.xfail(reason="https://github.com/nod-ai/SHARK/issues/311")`
	`53`	`+`
`51`	`54`	`self.module_tester.create_and_check_module(dynamic, device)`
`52`	`55`
`53`	`56`
Original file line number	Diff line number	Diff line change
`@@ -48,6 +48,10 @@ def test_module(self, dynamic, device):`
`48`	`48`	`pytest.skip(`
`49`	`49`	`reason="Dynamic tests not supported. mlir file not found."`
`50`	`50`	`)`
	`51`	`+`
	`52`	`+ if device in ["gpu", "cuda"]:`
	`53`	`+ pytest.xfail(reason="https://github.com/nod-ai/SHARK/issues/311")`
	`54`	`+`
`51`	`55`	`self.module_tester.create_and_check_module(dynamic, device)`
`52`	`56`
`53`	`57`