Merge branch 'zwei' into add-csv-serializer

Author: bveeramani

tests/conftest.py

@@ -197,16 +197,6 @@ def rl_ray_version(request):
     return request.param
 
 
-@pytest.fixture(scope="module")
-def chainer_full_version():
-    return "5.0.0"
-
-
-@pytest.fixture(scope="module")
-def chainer_full_py_version():
-    return "py3"
-
-
 @pytest.fixture(scope="module")
 def mxnet_full_version():
     return "1.6.0"
@@ -378,15 +368,28 @@ def _generate_all_framework_version_fixtures(metafunc):
     for fw in ("chainer", "tensorflow"):
         config = image_uris.config_for_framework(fw)
         if "scope" in config:
-            _parametrize_framework_version_fixture(metafunc, "{}_version".format(fw), config)
+            _parametrize_framework_version_fixtures(metafunc, fw, config)
         else:
             for image_scope in config.keys():
-                _parametrize_framework_version_fixture(
-                    metafunc, "{}_{}_version".format(fw, image_scope), config[image_scope]
+                _parametrize_framework_version_fixtures(
+                    metafunc, "{}_{}".format(fw, image_scope), config[image_scope]
                 )
 
 
-def _parametrize_framework_version_fixture(metafunc, fixture_name, config):
+def _parametrize_framework_version_fixtures(metafunc, fixture_prefix, config):
+    fixture_name = "{}_version".format(fixture_prefix)
     if fixture_name in metafunc.fixturenames:
         versions = list(config["versions"].keys()) + list(config.get("version_aliases", {}).keys())
         metafunc.parametrize(fixture_name, versions, scope="session")
+
+    latest_version = sorted(config["versions"].keys(), key=lambda v: Version(v))[-1]
+
+    fixture_name = "{}_latest_version".format(fixture_prefix)
+    if fixture_name in metafunc.fixturenames:
+        metafunc.parametrize(fixture_name, (latest_version,), scope="session")
+
+    fixture_name = "{}_latest_py_version".format(fixture_prefix)
+    if fixture_name in metafunc.fixturenames:
+        config = config["versions"]
+        py_versions = config[latest_version].get("py_versions", config[latest_version].keys())
+        metafunc.parametrize(fixture_name, (sorted(py_versions)[-1],), scope="session")

tests/integ/test_airflow_config.py

@@ -404,7 +404,7 @@ def test_rcf_airflow_config_uploads_data_source_to_s3(sagemaker_session, cpu_ins
 
 @pytest.mark.canary_quick
 def test_chainer_airflow_config_uploads_data_source_to_s3(
-    sagemaker_local_session, cpu_instance_type, chainer_full_version, chainer_full_py_version
+    sagemaker_local_session, cpu_instance_type, chainer_latest_version, chainer_latest_py_version
 ):
     with timeout(seconds=AIRFLOW_CONFIG_TIMEOUT_IN_SECONDS):
         script_path = os.path.join(DATA_DIR, "chainer_mnist", "mnist.py")
@@ -415,8 +415,8 @@ def test_chainer_airflow_config_uploads_data_source_to_s3(
             role=ROLE,
             instance_count=SINGLE_INSTANCE_COUNT,
             instance_type="local",
-            framework_version=chainer_full_version,
-            py_version=chainer_full_py_version,
+            framework_version=chainer_latest_version,
+            py_version=chainer_latest_py_version,
             sagemaker_session=sagemaker_local_session,
             hyperparameters={"epochs": 1},
             use_mpi=True,

tests/integ/test_chainer.py

@@ -26,25 +26,25 @@
 
 @pytest.fixture(scope="module")
 def chainer_local_training_job(
-    sagemaker_local_session, chainer_full_version, chainer_full_py_version
+    sagemaker_local_session, chainer_latest_version, chainer_latest_py_version
 ):
     return _run_mnist_training_job(
-        sagemaker_local_session, "local", 1, chainer_full_version, chainer_full_py_version
+        sagemaker_local_session, "local", 1, chainer_latest_version, chainer_latest_py_version
     )
 
 
 @pytest.mark.local_mode
 def test_distributed_cpu_training(
-    sagemaker_local_session, chainer_full_version, chainer_full_py_version
+    sagemaker_local_session, chainer_latest_version, chainer_latest_py_version
 ):
     _run_mnist_training_job(
-        sagemaker_local_session, "local", 2, chainer_full_version, chainer_full_py_version
+        sagemaker_local_session, "local", 2, chainer_latest_version, chainer_latest_py_version
     )
 
 
 @pytest.mark.local_mode
 def test_training_with_additional_hyperparameters(
-    sagemaker_local_session, chainer_full_version, chainer_full_py_version
+    sagemaker_local_session, chainer_latest_version, chainer_latest_py_version
 ):
     script_path = os.path.join(DATA_DIR, "chainer_mnist", "mnist.py")
     data_path = os.path.join(DATA_DIR, "chainer_mnist")
@@ -54,8 +54,8 @@ def test_training_with_additional_hyperparameters(
         role="SageMakerRole",
         instance_count=1,
         instance_type="local",
-        framework_version=chainer_full_version,
-        py_version=chainer_full_py_version,
+        framework_version=chainer_latest_version,
+        py_version=chainer_latest_py_version,
         sagemaker_session=sagemaker_local_session,
         hyperparameters={"epochs": 1},
         use_mpi=True,
@@ -72,7 +72,7 @@ def test_training_with_additional_hyperparameters(
 
 @pytest.mark.canary_quick
 def test_attach_deploy(
-    sagemaker_session, chainer_full_version, chainer_full_py_version, cpu_instance_type
+    sagemaker_session, chainer_latest_version, chainer_latest_py_version, cpu_instance_type
 ):
     with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
         script_path = os.path.join(DATA_DIR, "chainer_mnist", "mnist.py")
@@ -81,8 +81,8 @@ def test_attach_deploy(
         chainer = Chainer(
             entry_point=script_path,
             role="SageMakerRole",
-            framework_version=chainer_full_version,
-            py_version=chainer_full_py_version,
+            framework_version=chainer_latest_version,
+            py_version=chainer_latest_py_version,
             instance_count=1,
             instance_type=cpu_instance_type,
             sagemaker_session=sagemaker_session,
@@ -114,8 +114,8 @@ def test_attach_deploy(
 def test_deploy_model(
     chainer_local_training_job,
     sagemaker_local_session,
-    chainer_full_version,
-    chainer_full_py_version,
+    chainer_latest_version,
+    chainer_latest_py_version,
 ):
     script_path = os.path.join(DATA_DIR, "chainer_mnist", "mnist.py")
 
@@ -124,8 +124,8 @@ def test_deploy_model(
         "SageMakerRole",
         entry_point=script_path,
         sagemaker_session=sagemaker_local_session,
-        framework_version=chainer_full_version,
-        py_version=chainer_full_py_version,
+        framework_version=chainer_latest_version,
+        py_version=chainer_latest_py_version,
     )
 
     predictor = model.deploy(1, "local")

tests/integ/test_tuner.py

@@ -687,7 +687,7 @@ def test_tuning_tf_vpc_multi(
 
 @pytest.mark.canary_quick
 def test_tuning_chainer(
-    sagemaker_session, chainer_full_version, chainer_full_py_version, cpu_instance_type
+    sagemaker_session, chainer_latest_version, chainer_latest_py_version, cpu_instance_type
 ):
     with timeout(minutes=TUNING_DEFAULT_TIMEOUT_MINUTES):
         script_path = os.path.join(DATA_DIR, "chainer_mnist", "mnist.py")
@@ -696,8 +696,8 @@ def test_tuning_chainer(
         estimator = Chainer(
             entry_point=script_path,
             role="SageMakerRole",
-            framework_version=chainer_full_version,
-            py_version=chainer_full_py_version,
+            framework_version=chainer_latest_version,
+            py_version=chainer_latest_py_version,
             instance_count=1,
             instance_type=cpu_instance_type,
             sagemaker_session=sagemaker_session,