Porting model tests (#5622)

* Porting tests

* Remove unnecessary variable

* Fix linter

* Move prototype to extended tests

* Fix download models job

Author: datumbox

.circleci/config.yml

@@ -335,6 +335,20 @@ jobs:
           file_or_dir: test/test_onnx.py
 
   unittest_prototype:
+    docker:
+      - image: circleci/python:3.7
+    resource_class: xlarge
+    steps:
+      - checkout
+      - install_torchvision
+      - install_prototype_dependencies
+      - pip_install:
+          args: scipy pycocotools h5py
+          descr: Install optional dependencies
+      - run_tests_selective:
+          file_or_dir: test/test_prototype_*.py
+
+  unittest_extended:
     docker:
       - image: circleci/python:3.7
     resource_class: xlarge
@@ -346,18 +360,14 @@ jobs:
           command: |
             sudo apt update -qy && sudo apt install -qy parallel wget
             mkdir -p ~/.cache/torch/hub/checkpoints
-            python scripts/collect_model_urls.py torchvision/prototype/models \
+            python scripts/collect_model_urls.py torchvision/models \
                 | parallel -j0 'wget --no-verbose -O ~/.cache/torch/hub/checkpoints/`basename {}` {}\?source=ci'
       - install_torchvision
-      - install_prototype_dependencies
-      - pip_install:
-          args: scipy pycocotools h5py
-          descr: Install optional dependencies
       - run:
-          name: Enable prototype tests
-          command: echo 'export PYTORCH_TEST_WITH_PROTOTYPE=1' >> $BASH_ENV
+          name: Enable extended tests
+          command: echo 'export PYTORCH_TEST_WITH_EXTENDED=1' >> $BASH_ENV
       - run_tests_selective:
-          file_or_dir: test/test_prototype_*.py
+          file_or_dir: test/test_extended_*.py
 
   binary_linux_wheel:
     <<: *binary_common
@@ -1094,6 +1104,7 @@ workflows:
       - unittest_torchhub
       - unittest_onnx
       - unittest_prototype
+      - unittest_extended
       {{ unittest_workflows() }}
 
   cmake:

.circleci/config.yml.in

@@ -23,30 +23,30 @@ def get_available_models():
 @pytest.mark.parametrize("backbone_name", ("resnet18", "resnet50"))
 def test_resnet_fpn_backbone(backbone_name):
     x = torch.rand(1, 3, 300, 300, dtype=torch.float32, device="cpu")
-    model = resnet_fpn_backbone(backbone_name=backbone_name, pretrained=False)
+    model = resnet_fpn_backbone(backbone_name=backbone_name)
     assert isinstance(model, BackboneWithFPN)
     y = model(x)
     assert list(y.keys()) == ["0", "1", "2", "3", "pool"]
 
     with pytest.raises(ValueError, match=r"Trainable layers should be in the range"):
-        resnet_fpn_backbone(backbone_name=backbone_name, pretrained=False, trainable_layers=6)
+        resnet_fpn_backbone(backbone_name=backbone_name, trainable_layers=6)
     with pytest.raises(ValueError, match=r"Each returned layer should be in the range"):
-        resnet_fpn_backbone(backbone_name, False, returned_layers=[0, 1, 2, 3])
+        resnet_fpn_backbone(backbone_name=backbone_name, returned_layers=[0, 1, 2, 3])
     with pytest.raises(ValueError, match=r"Each returned layer should be in the range"):
-        resnet_fpn_backbone(backbone_name, False, returned_layers=[2, 3, 4, 5])
+        resnet_fpn_backbone(backbone_name=backbone_name, returned_layers=[2, 3, 4, 5])
 
 
 @pytest.mark.parametrize("backbone_name", ("mobilenet_v2", "mobilenet_v3_large", "mobilenet_v3_small"))
 def test_mobilenet_backbone(backbone_name):
     with pytest.raises(ValueError, match=r"Trainable layers should be in the range"):
-        mobilenet_backbone(backbone_name=backbone_name, pretrained=False, fpn=False, trainable_layers=-1)
+        mobilenet_backbone(backbone_name=backbone_name, fpn=False, trainable_layers=-1)
     with pytest.raises(ValueError, match=r"Each returned layer should be in the range"):
-        mobilenet_backbone(backbone_name, False, fpn=True, returned_layers=[-1, 0, 1, 2])
+        mobilenet_backbone(backbone_name=backbone_name, fpn=True, returned_layers=[-1, 0, 1, 2])
     with pytest.raises(ValueError, match=r"Each returned layer should be in the range"):
-        mobilenet_backbone(backbone_name, False, fpn=True, returned_layers=[3, 4, 5, 6])
-    model_fpn = mobilenet_backbone(backbone_name, False, fpn=True)
+        mobilenet_backbone(backbone_name=backbone_name, fpn=True, returned_layers=[3, 4, 5, 6])
+    model_fpn = mobilenet_backbone(backbone_name=backbone_name, fpn=True)
     assert isinstance(model_fpn, BackboneWithFPN)
-    model = mobilenet_backbone(backbone_name, False, fpn=False)
+    model = mobilenet_backbone(backbone_name=backbone_name, fpn=False)
     assert isinstance(model, torch.nn.Sequential)
 
 
@@ -100,7 +100,7 @@ def forward(self, x):
 
 class TestFxFeatureExtraction:
     inp = torch.rand(1, 3, 224, 224, dtype=torch.float32, device="cpu")
-    model_defaults = {"num_classes": 1, "pretrained": False}
+    model_defaults = {"num_classes": 1}
     leaf_modules = []
 
     def _create_feature_extractor(self, *args, **kwargs):

test/test_backbone_utils.py

@@ -53,50 +53,49 @@ def read_image2():
     "see https://github.com/pytorch/vision/issues/1191",
 )
 class Tester(unittest.TestCase):
-    pretrained = False
     image = read_image1()
 
     def test_alexnet(self):
-        process_model(models.alexnet(self.pretrained), self.image, _C_tests.forward_alexnet, "Alexnet")
+        process_model(models.alexnet(), self.image, _C_tests.forward_alexnet, "Alexnet")
 
     def test_vgg11(self):
-        process_model(models.vgg11(self.pretrained), self.image, _C_tests.forward_vgg11, "VGG11")
+        process_model(models.vgg11(), self.image, _C_tests.forward_vgg11, "VGG11")
 
     def test_vgg13(self):
-        process_model(models.vgg13(self.pretrained), self.image, _C_tests.forward_vgg13, "VGG13")
+        process_model(models.vgg13(), self.image, _C_tests.forward_vgg13, "VGG13")
 
     def test_vgg16(self):
-        process_model(models.vgg16(self.pretrained), self.image, _C_tests.forward_vgg16, "VGG16")
+        process_model(models.vgg16(), self.image, _C_tests.forward_vgg16, "VGG16")
 
     def test_vgg19(self):
-        process_model(models.vgg19(self.pretrained), self.image, _C_tests.forward_vgg19, "VGG19")
+        process_model(models.vgg19(), self.image, _C_tests.forward_vgg19, "VGG19")
 
     def test_vgg11_bn(self):
-        process_model(models.vgg11_bn(self.pretrained), self.image, _C_tests.forward_vgg11bn, "VGG11BN")
+        process_model(models.vgg11_bn(), self.image, _C_tests.forward_vgg11bn, "VGG11BN")
 
     def test_vgg13_bn(self):
-        process_model(models.vgg13_bn(self.pretrained), self.image, _C_tests.forward_vgg13bn, "VGG13BN")
+        process_model(models.vgg13_bn(), self.image, _C_tests.forward_vgg13bn, "VGG13BN")
 
     def test_vgg16_bn(self):
-        process_model(models.vgg16_bn(self.pretrained), self.image, _C_tests.forward_vgg16bn, "VGG16BN")
+        process_model(models.vgg16_bn(), self.image, _C_tests.forward_vgg16bn, "VGG16BN")
 
     def test_vgg19_bn(self):
-        process_model(models.vgg19_bn(self.pretrained), self.image, _C_tests.forward_vgg19bn, "VGG19BN")
+        process_model(models.vgg19_bn(), self.image, _C_tests.forward_vgg19bn, "VGG19BN")
 
     def test_resnet18(self):
-        process_model(models.resnet18(self.pretrained), self.image, _C_tests.forward_resnet18, "Resnet18")
+        process_model(models.resnet18(), self.image, _C_tests.forward_resnet18, "Resnet18")
 
     def test_resnet34(self):
-        process_model(models.resnet34(self.pretrained), self.image, _C_tests.forward_resnet34, "Resnet34")
+        process_model(models.resnet34(), self.image, _C_tests.forward_resnet34, "Resnet34")
 
     def test_resnet50(self):
-        process_model(models.resnet50(self.pretrained), self.image, _C_tests.forward_resnet50, "Resnet50")
+        process_model(models.resnet50(), self.image, _C_tests.forward_resnet50, "Resnet50")
 
     def test_resnet101(self):
-        process_model(models.resnet101(self.pretrained), self.image, _C_tests.forward_resnet101, "Resnet101")
+        process_model(models.resnet101(), self.image, _C_tests.forward_resnet101, "Resnet101")
 
     def test_resnet152(self):
-        process_model(models.resnet152(self.pretrained), self.image, _C_tests.forward_resnet152, "Resnet152")
+        process_model(models.resnet152(), self.image, _C_tests.forward_resnet152, "Resnet152")
 
     def test_resnext50_32x4d(self):
         process_model(models.resnext50_32x4d(), self.image, _C_tests.forward_resnext50_32x4d, "ResNext50_32x4d")
@@ -111,48 +110,44 @@ def test_wide_resnet101_2(self):
         process_model(models.wide_resnet101_2(), self.image, _C_tests.forward_wide_resnet101_2, "WideResNet101_2")
 
     def test_squeezenet1_0(self):
-        process_model(
-            models.squeezenet1_0(self.pretrained), self.image, _C_tests.forward_squeezenet1_0, "Squeezenet1.0"
-        )
+        process_model(models.squeezenet1_0(), self.image, _C_tests.forward_squeezenet1_0, "Squeezenet1.0")
 
     def test_squeezenet1_1(self):
-        process_model(
-            models.squeezenet1_1(self.pretrained), self.image, _C_tests.forward_squeezenet1_1, "Squeezenet1.1"
-        )
+        process_model(models.squeezenet1_1(), self.image, _C_tests.forward_squeezenet1_1, "Squeezenet1.1")
 
     def test_densenet121(self):
-        process_model(models.densenet121(self.pretrained), self.image, _C_tests.forward_densenet121, "Densenet121")
+        process_model(models.densenet121(), self.image, _C_tests.forward_densenet121, "Densenet121")
 
     def test_densenet169(self):
-        process_model(models.densenet169(self.pretrained), self.image, _C_tests.forward_densenet169, "Densenet169")
+        process_model(models.densenet169(), self.image, _C_tests.forward_densenet169, "Densenet169")
 
     def test_densenet201(self):
-        process_model(models.densenet201(self.pretrained), self.image, _C_tests.forward_densenet201, "Densenet201")
+        process_model(models.densenet201(), self.image, _C_tests.forward_densenet201, "Densenet201")
 
     def test_densenet161(self):
-        process_model(models.densenet161(self.pretrained), self.image, _C_tests.forward_densenet161, "Densenet161")
+        process_model(models.densenet161(), self.image, _C_tests.forward_densenet161, "Densenet161")
 
     def test_mobilenet_v2(self):
-        process_model(models.mobilenet_v2(self.pretrained), self.image, _C_tests.forward_mobilenetv2, "MobileNet")
+        process_model(models.mobilenet_v2(), self.image, _C_tests.forward_mobilenetv2, "MobileNet")
 
     def test_googlenet(self):
-        process_model(models.googlenet(self.pretrained), self.image, _C_tests.forward_googlenet, "GoogLeNet")
+        process_model(models.googlenet(), self.image, _C_tests.forward_googlenet, "GoogLeNet")
 
     def test_mnasnet0_5(self):
-        process_model(models.mnasnet0_5(self.pretrained), self.image, _C_tests.forward_mnasnet0_5, "MNASNet0_5")
+        process_model(models.mnasnet0_5(), self.image, _C_tests.forward_mnasnet0_5, "MNASNet0_5")
 
     def test_mnasnet0_75(self):
-        process_model(models.mnasnet0_75(self.pretrained), self.image, _C_tests.forward_mnasnet0_75, "MNASNet0_75")
+        process_model(models.mnasnet0_75(), self.image, _C_tests.forward_mnasnet0_75, "MNASNet0_75")
 
     def test_mnasnet1_0(self):
-        process_model(models.mnasnet1_0(self.pretrained), self.image, _C_tests.forward_mnasnet1_0, "MNASNet1_0")
+        process_model(models.mnasnet1_0(), self.image, _C_tests.forward_mnasnet1_0, "MNASNet1_0")
 
     def test_mnasnet1_3(self):
-        process_model(models.mnasnet1_3(self.pretrained), self.image, _C_tests.forward_mnasnet1_3, "MNASNet1_3")
+        process_model(models.mnasnet1_3(), self.image, _C_tests.forward_mnasnet1_3, "MNASNet1_3")
 
     def test_inception_v3(self):
         self.image = read_image2()
-        process_model(models.inception_v3(self.pretrained), self.image, _C_tests.forward_inceptionv3, "Inceptionv3")
+        process_model(models.inception_v3(), self.image, _C_tests.forward_inceptionv3, "Inceptionv3")
 
 
 if __name__ == "__main__":

test/test_cpp_models.py

@@ -3,22 +3,16 @@
 
 import pytest
 import test_models as TM
-import torchvision
+from torchvision import models
 from torchvision.models._api import WeightsEnum, Weights
 from torchvision.models._utils import handle_legacy_interface
 
 run_if_test_with_prototype = pytest.mark.skipif(
-    os.getenv("PYTORCH_TEST_WITH_PROTOTYPE") != "1",
-    reason="Prototype tests are disabled by default. Set PYTORCH_TEST_WITH_PROTOTYPE=1 to run them.",
+    os.getenv("PYTORCH_TEST_WITH_EXTENDED") != "1",
+    reason="Extended tests are disabled by default. Set PYTORCH_TEST_WITH_EXTENDED=1 to run them.",
 )
 
 
-def _get_original_model(model_fn):
-    original_module_name = model_fn.__module__.replace(".prototype", "")
-    module = importlib.import_module(original_module_name)
-    return module.__dict__[model_fn.__name__]
-
-
 def _get_parent_module(model_fn):
     parent_module_name = ".".join(model_fn.__module__.split(".")[:-1])
     module = importlib.import_module(parent_module_name)
@@ -38,44 +32,33 @@ def _get_model_weights(model_fn):
         return None
 
 
-def _build_model(fn, **kwargs):
-    try:
-        model = fn(**kwargs)
-    except ValueError as e:
-        msg = str(e)
-        if "No checkpoint is available" in msg:
-            pytest.skip(msg)
-        raise e
-    return model.eval()
-
-
 @pytest.mark.parametrize(
     "name, weight",
     [
-        ("ResNet50_Weights.IMAGENET1K_V1", torchvision.models.ResNet50_Weights.IMAGENET1K_V1),
-        ("ResNet50_Weights.DEFAULT", torchvision.models.ResNet50_Weights.IMAGENET1K_V2),
+        ("ResNet50_Weights.IMAGENET1K_V1", models.ResNet50_Weights.IMAGENET1K_V1),
+        ("ResNet50_Weights.DEFAULT", models.ResNet50_Weights.IMAGENET1K_V2),
         (
             "ResNet50_QuantizedWeights.DEFAULT",
-            torchvision.models.quantization.ResNet50_QuantizedWeights.IMAGENET1K_FBGEMM_V2,
+            models.quantization.ResNet50_QuantizedWeights.IMAGENET1K_FBGEMM_V2,
         ),
         (
             "ResNet50_QuantizedWeights.IMAGENET1K_FBGEMM_V1",
-            torchvision.models.quantization.ResNet50_QuantizedWeights.IMAGENET1K_FBGEMM_V1,
+            models.quantization.ResNet50_QuantizedWeights.IMAGENET1K_FBGEMM_V1,
         ),
     ],
 )
 def test_get_weight(name, weight):
-    assert torchvision.models.get_weight(name) == weight
+    assert models.get_weight(name) == weight
 
 
 @pytest.mark.parametrize(
     "model_fn",
-    TM.get_models_from_module(torchvision.models)
-    + TM.get_models_from_module(torchvision.models.detection)
-    + TM.get_models_from_module(torchvision.models.quantization)
-    + TM.get_models_from_module(torchvision.models.segmentation)
-    + TM.get_models_from_module(torchvision.models.video)
-    + TM.get_models_from_module(torchvision.models.optical_flow),
+    TM.get_models_from_module(models)
+    + TM.get_models_from_module(models.detection)
+    + TM.get_models_from_module(models.quantization)
+    + TM.get_models_from_module(models.segmentation)
+    + TM.get_models_from_module(models.video)
+    + TM.get_models_from_module(models.optical_flow),
 )
 def test_naming_conventions(model_fn):
     weights_enum = _get_model_weights(model_fn)
@@ -86,12 +69,12 @@ def test_naming_conventions(model_fn):
 
 @pytest.mark.parametrize(
     "model_fn",
-    TM.get_models_from_module(torchvision.models)
-    + TM.get_models_from_module(torchvision.models.detection)
-    + TM.get_models_from_module(torchvision.models.quantization)
-    + TM.get_models_from_module(torchvision.models.segmentation)
-    + TM.get_models_from_module(torchvision.models.video)
-    + TM.get_models_from_module(torchvision.models.optical_flow),
+    TM.get_models_from_module(models)
+    + TM.get_models_from_module(models.detection)
+    + TM.get_models_from_module(models.quantization)
+    + TM.get_models_from_module(models.segmentation)
+    + TM.get_models_from_module(models.video)
+    + TM.get_models_from_module(models.optical_flow),
 )
 @run_if_test_with_prototype
 def test_schema_meta_validation(model_fn):

test/test_prototype_models.py renamed to test/test_extended_models.py

@@ -26,13 +26,13 @@ class TestHub:
     #   Python cache as we run all hub tests in the same python process.
 
     def test_load_from_github(self):
-        hub_model = hub.load("pytorch/vision", "resnet18", pretrained=True, progress=False)
+        hub_model = hub.load("pytorch/vision", "resnet18", weights="DEFAULT", progress=False)
         assert sum_of_model_parameters(hub_model).item() == pytest.approx(SUM_OF_PRETRAINED_RESNET18_PARAMS)
 
     def test_set_dir(self):
         temp_dir = tempfile.gettempdir()
         hub.set_dir(temp_dir)
-        hub_model = hub.load("pytorch/vision", "resnet18", pretrained=True, progress=False)
+        hub_model = hub.load("pytorch/vision", "resnet18", weights="DEFAULT", progress=False)
         assert sum_of_model_parameters(hub_model).item() == pytest.approx(SUM_OF_PRETRAINED_RESNET18_PARAMS)
         assert os.path.exists(temp_dir + "/pytorch_vision_master")
         shutil.rmtree(temp_dir + "/pytorch_vision_master")