change: allow ModelMonitor and Processor to take IAM role names (in addition to ARNs)

This change also removes some hardcoded AWS account IDs and regions
from some integration tests.

Author: laurenyu

src/sagemaker/automl/automl.py

@@ -10,7 +10,7 @@
 # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
-"""A class for SageMaker AutoML Job."""
+"""A class for SageMaker AutoML Jobs."""
 from __future__ import absolute_import
 
 from six import string_types

src/sagemaker/model_monitor/model_monitoring.py

@@ -22,24 +22,17 @@
 import logging
 import uuid
 
-from six.moves.urllib.parse import urlparse
 from six import string_types
-
+from six.moves.urllib.parse import urlparse
 from botocore.exceptions import ClientError
 
+from sagemaker.exceptions import UnexpectedStatusException
+from sagemaker.model_monitor.monitoring_files import Constraints, ConstraintViolations, Statistics
 from sagemaker.network import NetworkConfig
+from sagemaker.processing import Processor, ProcessingInput, ProcessingJob, ProcessingOutput
 from sagemaker.s3 import S3Uploader
-
-from sagemaker.utils import name_from_base
 from sagemaker.session import Session
-from sagemaker.processing import Processor
-from sagemaker.processing import ProcessingJob
-from sagemaker.processing import ProcessingInput
-from sagemaker.processing import ProcessingOutput
-from sagemaker.model_monitor.monitoring_files import Constraints, ConstraintViolations
-from sagemaker.model_monitor.monitoring_files import Statistics
-from sagemaker.exceptions import UnexpectedStatusException
-from sagemaker.utils import retries
+from sagemaker.utils import name_from_base, retries
 
 _DEFAULT_MONITOR_IMAGE_URI_WITH_PLACEHOLDERS = (
     "{}.dkr.ecr.{}.amazonaws.com/sagemaker-model-monitor-analyzer"
@@ -390,7 +383,7 @@ def update_monitoring_schedule(
             network_config (sagemaker.network.NetworkConfig): A NetworkConfig
                 object that configures network isolation, encryption of
                 inter-container traffic, security group IDs, and subnets.
-            role (str): An AWS IAM role. The Amazon SageMaker jobs use this role.
+            role (str): An AWS IAM role name or ARN. The Amazon SageMaker jobs use this role.
             image_uri (str): The uri of the image to use for the jobs started by
                 the Monitor.
 
@@ -452,7 +445,7 @@ def update_monitoring_schedule(
             self.network_config = network_config
 
         if role is not None:
-            self.role = role
+            self.role = self.sagemaker_session.expand_role(role)
 
         if image_uri is not None:
             self.image_uri = image_uri
@@ -988,7 +981,7 @@ def __init__(
         creating Amazon SageMaker Monitoring Schedules to monitor SageMaker endpoints.
 
         Args:
-            role (str): An AWS IAM role. The Amazon SageMaker jobs use this role.
+            role (str): An AWS IAM role name or ARN. The Amazon SageMaker jobs use this role.
             instance_count (int): The number of instances to run the jobs with.
             instance_type (str): Type of EC2 instance to use for the job, for example,
                 'ml.m5.xlarge'.
@@ -1355,7 +1348,7 @@ def update_monitoring_schedule(
                 inter-container traffic, security group IDs, and subnets.
             enable_cloudwatch_metrics (bool): Whether to publish cloudwatch metrics as part of
                 the baselining or monitoring jobs.
-            role (str): An AWS IAM role. The Amazon SageMaker jobs use this role.
+            role (str): An AWS IAM role name or ARN. The Amazon SageMaker jobs use this role.
 
         """
         monitoring_inputs = None
@@ -1431,7 +1424,7 @@ def update_monitoring_schedule(
             network_config_dict = self.network_config._to_request_dict()
 
         if role is not None:
-            self.role = role
+            self.role = self.sagemaker_session.expand_role(role)
 
         self.sagemaker_session.update_monitoring_schedule(
             monitoring_schedule_name=self.monitoring_schedule_name,

src/sagemaker/processing.py

@@ -52,7 +52,7 @@ def __init__(
         SageMaker processing tasks.
 
         Args:
-            role (str): An AWS IAM role. The Amazon SageMaker training jobs
+            role (str): An AWS IAM role name or ARN. The Amazon SageMaker training jobs
                 and APIs that create Amazon SageMaker endpoints use this role
                 to access training data and model artifacts. After the endpoint
                 is created, the inference code might use the IAM role, if it
@@ -281,7 +281,7 @@ def __init__(
         handles Amazon SageMaker processing tasks for jobs using script mode.
 
         Args:
-            role (str): An AWS IAM role. The Amazon SageMaker training jobs
+            role (str): An AWS IAM role name or ARN. The Amazon SageMaker training jobs
                 and APIs that create Amazon SageMaker endpoints use this role
                 to access training data and model artifacts. After the endpoint
                 is created, the inference code might use the IAM role, if it
@@ -538,7 +538,7 @@ def start_new(cls, processor, inputs, outputs, experiment_config):
         else:
             process_request_args["network_config"] = None
 
-        process_request_args["role_arn"] = processor.role
+        process_request_args["role_arn"] = processor.sagemaker_session.expand_role(processor.role)
 
         process_request_args["tags"] = processor.tags
 

tests/integ/test_auto_ml.py

@@ -23,7 +23,6 @@
 from tests.integ import DATA_DIR, AUTO_ML_DEFAULT_TIMEMOUT_MINUTES
 from tests.integ.timeout import timeout
 
-DEV_ACCOUNT = 142577830533
 ROLE = "SageMakerRole"
 PREFIX = "sagemaker/beta-automl-xgboost"
 HOSTING_INSTANCE_TYPE = "ml.c4.xlarge"
@@ -40,26 +39,10 @@
 # use a succeeded AutoML job to test describe and list candidates method, otherwise tests will run too long
 AUTO_ML_JOB_NAME = "sagemaker-auto-gamma-ml-test"
 
-EXPECTED_DEFAULT_INPUT_CONFIG = [
-    {
-        "DataSource": {
-            "S3DataSource": {
-                "S3DataType": "S3Prefix",
-                "S3Uri": "s3://sagemaker-us-east-2-{}/{}/input/iris_training.csv".format(
-                    DEV_ACCOUNT, PREFIX
-                ),
-            }
-        },
-        "TargetAttributeName": TARGET_ATTRIBUTE_NAME,
-    }
-]
 EXPECTED_DEFAULT_JOB_CONFIG = {
     "CompletionCriteria": {"MaxCandidates": 3},
     "SecurityConfig": {"EnableInterContainerTrafficEncryption": False},
 }
-EXPECTED_DEFAULT_OUTPUT_CONFIG = {
-    "S3OutputPath": "s3://sagemaker-us-east-2-{}/".format(DEV_ACCOUNT)
-}
 
 
 def test_auto_ml_fit(sagemaker_session):
@@ -102,7 +85,7 @@ def test_auto_ml_input_object_fit(sagemaker_session):
 
 
 def test_auto_ml_fit_optional_args(sagemaker_session):
-    output_path = "s3://sagemaker-us-east-2-{}/{}".format(DEV_ACCOUNT, "specified_ouput_path")
+    output_path = "s3://{}/{}".format(sagemaker_session.default_bucket(), "specified_ouput_path")
     problem_type = "MulticlassClassification"
     job_objective = {"MetricName": "Accuracy"}
     auto_ml = AutoML(
@@ -138,6 +121,23 @@ def test_auto_ml_invalid_target_attribute(sagemaker_session):
 
 
 def test_auto_ml_describe_auto_ml_job(sagemaker_session):
+    expected_default_input_config = [
+        {
+            "DataSource": {
+                "S3DataSource": {
+                    "S3DataType": "S3Prefix",
+                    "S3Uri": "s3://{}/{}/input/iris_training.csv".format(
+                        sagemaker_session.default_bucket(), PREFIX
+                    ),
+                }
+            },
+            "TargetAttributeName": TARGET_ATTRIBUTE_NAME,
+        }
+    ]
+    expected_default_output_config = {
+        "S3OutputPath": "s3://{}/".format(sagemaker_session.default_bucket())
+    }
+
     auto_ml = AutoML(
         role=ROLE, target_attribute_name=TARGET_ATTRIBUTE_NAME, sagemaker_session=sagemaker_session
     )
@@ -146,9 +146,9 @@ def test_auto_ml_describe_auto_ml_job(sagemaker_session):
     assert desc["AutoMLJobName"] == AUTO_ML_JOB_NAME
     assert desc["AutoMLJobStatus"] == "Completed"
     assert isinstance(desc["BestCandidate"], dict)
-    assert desc["InputDataConfig"] == EXPECTED_DEFAULT_INPUT_CONFIG
+    assert desc["InputDataConfig"] == expected_default_input_config
     assert desc["AutoMLJobConfig"] == EXPECTED_DEFAULT_JOB_CONFIG
-    assert desc["OutputDataConfig"] == EXPECTED_DEFAULT_OUTPUT_CONFIG
+    assert desc["OutputDataConfig"] == expected_default_output_config
 
 
 def test_list_candidates(sagemaker_session):

tests/integ/test_model_monitor.py

@@ -42,7 +42,7 @@
 from tests.integ.kms_utils import get_or_create_kms_key
 from tests.integ.retry import retries
 
-ROLE = "arn:aws:iam::142577830533:role/SageMakerRole"
+ROLE = "SageMakerRole"
 INSTANCE_COUNT = 1
 INSTANCE_TYPE = "ml.m5.xlarge"
 VOLUME_SIZE_IN_GB = 40
@@ -63,7 +63,7 @@
 DEFAULT_EXECUTION_MAX_RUNTIME_IN_SECONDS = 3600
 DEFAULT_IMAGE_SUFFIX = ".com/sagemaker-model-monitor-analyzer"
 
-UPDATED_ROLE = "arn:aws:iam::142577830533:role/SageMakerRole"
+UPDATED_ROLE = "SageMakerRole"
 UPDATED_INSTANCE_COUNT = 2
 UPDATED_INSTANCE_TYPE = "ml.m5.2xlarge"
 UPDATED_VOLUME_SIZE_IN_GB = 50
@@ -99,7 +99,7 @@ def predictor(sagemaker_session, tf_full_version):
     ):
         model = Model(
             model_data=model_data,
-            role="SageMakerRole",
+            role=ROLE,
             framework_version=tf_full_version,
             sagemaker_session=sagemaker_session,
         )

tests/integ/test_processing.py

@@ -23,7 +23,7 @@
 from tests.integ import DATA_DIR
 from tests.integ.kms_utils import get_or_create_kms_key
 
-ROLE = "arn:aws:iam::142577830533:role/SageMakerRole"
+ROLE = "SageMakerRole"
 
 
 @pytest.fixture(scope="module")
@@ -57,8 +57,6 @@ def output_kms_key(sagemaker_session):
 
 
 def test_sklearn(sagemaker_session, sklearn_full_version, cpu_instance_type):
-    logging.getLogger().setLevel(logging.DEBUG)  # TODO-reinvent-2019: REMOVE
-
     script_path = os.path.join(DATA_DIR, "dummy_script.py")
     input_file_path = os.path.join(DATA_DIR, "dummy_input.txt")