change: copy all tests to test-toolkit folder. (#292)

Author: nadiaya

.flake8

@@ -1,3 +1,3 @@
 [flake8]
-application_import_names = sagemaker_tensorflow_container, test, timeout, utils
+application_import_names = sagemaker_tensorflow_container, test, test-toolkit, timeout, utils
 import-order-style = google

test-toolkit/__init__.py

@@ -0,0 +1,13 @@
+#  Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License").
+#  You may not use this file except in compliance with the License.
+#  A copy of the License is located at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  or in the "license" file accompanying this file. This file is distributed
+#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+#  express or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+from __future__ import absolute_import

test-toolkit/integration/__init__.py

@@ -0,0 +1,48 @@
+#  Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License").
+#  You may not use this file except in compliance with the License.
+#  A copy of the License is located at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  or in the "license" file accompanying this file. This file is distributed
+#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+#  express or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+from __future__ import absolute_import
+
+import logging
+import os
+
+logging.getLogger('boto3').setLevel(logging.INFO)
+logging.getLogger('botocore').setLevel(logging.INFO)
+
+RESOURCE_PATH = os.path.join(os.path.dirname(__file__), '..', 'resources')
+
+# these regions have some p2 and p3 instances, but not enough for automated testing
+NO_P2_REGIONS = [
+    'ca-central-1',
+    'eu-central-1',
+    'eu-west-2',
+    'us-west-1',
+    'eu-west-3',
+    'eu-north-1',
+    'sa-east-1',
+    'ap-east-1',
+    'me-south-1'
+]
+NO_P3_REGIONS = [
+    'ap-southeast-1',
+    'ap-southeast-2',
+    'ap-south-1',
+    'ca-central-1',
+    'eu-central-1',
+    'eu-west-2',
+    'us-west-1'
+    'eu-west-3',
+    'eu-north-1',
+    'sa-east-1',
+    'ap-east-1',
+    'me-south-1'
+]

test-toolkit/integration/conftest.py

@@ -0,0 +1,118 @@
+#  Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License").
+#  You may not use this file except in compliance with the License.
+#  A copy of the License is located at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  or in the "license" file accompanying this file. This file is distributed
+#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+#  express or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+from __future__ import absolute_import
+
+import logging
+import os
+
+import boto3
+import pytest
+from sagemaker import LocalSession, Session
+from sagemaker.tensorflow import TensorFlow
+
+from test.integration import NO_P2_REGIONS, NO_P3_REGIONS
+
+logger = logging.getLogger(__name__)
+logging.getLogger('boto').setLevel(logging.INFO)
+logging.getLogger('botocore').setLevel(logging.INFO)
+logging.getLogger('factory.py').setLevel(logging.INFO)
+logging.getLogger('auth.py').setLevel(logging.INFO)
+logging.getLogger('connectionpool.py').setLevel(logging.INFO)
+
+SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__))
+
+
+def pytest_addoption(parser):
+    parser.addoption('--docker-base-name', default='sagemaker-tensorflow-scriptmode')
+    parser.addoption('--tag', default=None)
+    parser.addoption('--region', default='us-west-2')
+    parser.addoption('--framework-version', default=TensorFlow.LATEST_VERSION)
+    parser.addoption('--processor', default='cpu', choices=['cpu', 'gpu', 'cpu,gpu'])
+    parser.addoption('--py-version', default='3', choices=['2', '3', '2,3'])
+    parser.addoption('--account-id', default='142577830533')
+    parser.addoption('--instance-type', default=None)
+
+
+def pytest_configure(config):
+    os.environ['TEST_PY_VERSIONS'] = config.getoption('--py-version')
+    os.environ['TEST_PROCESSORS'] = config.getoption('--processor')
+
+
+@pytest.fixture(scope='session')
+def docker_base_name(request):
+    return request.config.getoption('--docker-base-name')
+
+
+@pytest.fixture(scope='session')
+def region(request):
+    return request.config.getoption('--region')
+
+
+@pytest.fixture(scope='session')
+def framework_version(request):
+    return request.config.getoption('--framework-version')
+
+
+@pytest.fixture
+def tag(request, framework_version, processor, py_version):
+    provided_tag = request.config.getoption('--tag')
+    default_tag = '{}-{}-py{}'.format(framework_version, processor, py_version)
+    return provided_tag if provided_tag is not None else default_tag
+
+
+@pytest.fixture(scope='session')
+def sagemaker_session(region):
+    return Session(boto_session=boto3.Session(region_name=region))
+
+
+@pytest.fixture(scope='session')
+def sagemaker_local_session(region):
+    return LocalSession(boto_session=boto3.Session(region_name=region))
+
+
+@pytest.fixture(scope='session')
+def account_id(request):
+    return request.config.getoption('--account-id')
+
+
+@pytest.fixture
+def instance_type(request, processor):
+    provided_instance_type = request.config.getoption('--instance-type')
+    default_instance_type = 'ml.c4.xlarge' if processor == 'cpu' else 'ml.p2.xlarge'
+    return provided_instance_type if provided_instance_type is not None else default_instance_type
+
+
+@pytest.fixture(autouse=True)
+def skip_by_device_type(request, processor):
+    is_gpu = (processor == 'gpu')
+    if (request.node.get_closest_marker('skip_gpu') and is_gpu) or \
+            (request.node.get_closest_marker('skip_cpu') and not is_gpu):
+        pytest.skip('Skipping because running on \'{}\' instance'.format(processor))
+
+
+@pytest.fixture(autouse=True)
+def skip_gpu_instance_restricted_regions(region, instance_type):
+    if (region in NO_P2_REGIONS and instance_type.startswith('ml.p2')) or \
+            (region in NO_P3_REGIONS and instance_type.startswith('ml.p3')):
+        pytest.skip('Skipping GPU test in region {}'.format(region))
+
+
+@pytest.fixture
+def docker_image(docker_base_name, tag):
+    return '{}:{}'.format(docker_base_name, tag)
+
+
+@pytest.fixture
+def ecr_image(account_id, docker_base_name, tag, region):
+    return '{}.dkr.ecr.{}.amazonaws.com/{}:{}'.format(
+        account_id, region, docker_base_name, tag)

test-toolkit/integration/local/test_horovod.py

@@ -0,0 +1,82 @@
+# Copyright 2017-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+from __future__ import absolute_import
+
+import json
+import os
+import tarfile
+
+import pytest
+from sagemaker.tensorflow import TensorFlow
+
+from test.integration.utils import processor, py_version  # noqa: F401
+
+RESOURCE_PATH = os.path.join(os.path.dirname(__file__), '..', '..', 'resources')
+
+
+@pytest.mark.skip_gpu
+@pytest.mark.parametrize('instances, processes', [
+    [1, 2],
+    (2, 1),
+    (2, 2),
+    (5, 2)])
+def test_distributed_training_horovod_basic(instances,
+                                            processes,
+                                            sagemaker_local_session,
+                                            docker_image,
+                                            tmpdir,
+                                            framework_version):
+    output_path = 'file://%s' % tmpdir
+    estimator = TensorFlow(
+        entry_point=os.path.join(RESOURCE_PATH, 'hvdbasic', 'train_hvd_basic.py'),
+        role='SageMakerRole',
+        train_instance_type='local',
+        sagemaker_session=sagemaker_local_session,
+        train_instance_count=instances,
+        image_name=docker_image,
+        output_path=output_path,
+        framework_version=framework_version,
+        hyperparameters={'sagemaker_mpi_enabled': True,
+                         'sagemaker_network_interface_name': 'eth0',
+                         'sagemaker_mpi_num_of_processes_per_host': processes})
+
+    estimator.fit('file://{}'.format(os.path.join(RESOURCE_PATH, 'mnist', 'data-distributed')))
+
+    tmp = str(tmpdir)
+    extract_files(output_path.replace('file://', ''), tmp)
+
+    size = instances * processes
+
+    for rank in range(size):
+        local_rank = rank % processes
+        assert read_json('local-rank-%s-rank-%s' % (local_rank, rank), tmp) == {
+            'local-rank': local_rank, 'rank': rank, 'size': size}
+
+
+def read_json(file, tmp):
+    with open(os.path.join(tmp, file)) as f:
+        return json.load(f)
+
+
+def assert_files_exist_in_tar(output_path, files):
+    if output_path.startswith('file://'):
+        output_path = output_path[7:]
+    model_file = os.path.join(output_path, 'model.tar.gz')
+    with tarfile.open(model_file) as tar:
+        for f in files:
+            tar.getmember(f)
+
+
+def extract_files(output_path, tmpdir):
+    with tarfile.open(os.path.join(output_path, 'model.tar.gz')) as tar:
+        tar.extractall(tmpdir)

test-toolkit/integration/local/test_keras.py

@@ -0,0 +1,57 @@
+# Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+from __future__ import absolute_import
+
+import logging
+import os
+
+import numpy as np
+import pytest
+from sagemaker.tensorflow import serving, TensorFlow
+
+from test.integration import RESOURCE_PATH
+from test.integration.utils import processor, py_version  # noqa: F401
+
+
+logging.basicConfig(level=logging.DEBUG)
+
+
+@pytest.mark.skip(reason="Serving part fails because of version mismatch.")
+def test_keras_training(sagemaker_local_session, docker_image, tmpdir, framework_version):
+    entry_point = os.path.join(RESOURCE_PATH, 'keras_inception.py')
+    output_path = 'file://{}'.format(tmpdir)
+
+    estimator = TensorFlow(
+        entry_point=entry_point,
+        role='SageMakerRole',
+        train_instance_count=1,
+        train_instance_type='local',
+        image_name=docker_image,
+        sagemaker_session=sagemaker_local_session,
+        model_dir='/opt/ml/model',
+        output_path=output_path,
+        framework_version=framework_version,
+        py_version='py3')
+
+    estimator.fit()
+
+    model = serving.Model(model_data=output_path,
+                          role='SageMakerRole',
+                          framework_version=framework_version,
+                          sagemaker_session=sagemaker_local_session)
+
+    predictor = model.deploy(initial_instance_count=1, instance_type='local')
+
+    assert predictor.predict(np.random.randn(4, 4, 4, 2) * 255)
+
+    predictor.delete_endpoint()