change: Implement test mechanism for Pipeline variables

annotations for processors + estimators / test mechanism change

annotations for processors + estimators / test mechanism change

remove debug print

reformatting

update TM and resolve all AIs and all untested subclasses

Add ppl var annotation to all composite object for training

Adjust tm for recent model changes

Author: Dewen Qi

src/sagemaker/amazon/amazon_estimator.py

@@ -18,7 +18,6 @@
 import json
 import logging
 import tempfile
-from typing import Union
 
 from six.moves.urllib.parse import urlparse
 
@@ -32,7 +31,6 @@
 from sagemaker.utils import sagemaker_timestamp
 from sagemaker.workflow.entities import PipelineVariable
 from sagemaker.workflow.pipeline_context import runnable_by_pipeline
-from sagemaker.workflow.entities import PipelineVariable
 from sagemaker.workflow.parameters import ParameterBoolean
 from sagemaker.workflow import is_pipeline_variable
 

src/sagemaker/amazon/hyperparameter.py

@@ -16,8 +16,6 @@
 import json
 from sagemaker.workflow import is_pipeline_variable
 
-from sagemaker.workflow import is_pipeline_variable
-
 
 class Hyperparameter(object):
     """An algorithm hyperparameter with optional validation.

src/sagemaker/amazon/kmeans.py

@@ -27,8 +27,6 @@
 from sagemaker.vpc_utils import VPC_CONFIG_DEFAULT
 from sagemaker.workflow.entities import PipelineVariable
 
-from sagemaker.workflow.entities import PipelineVariable
-
 
 class KMeans(AmazonAlgorithmEstimatorBase):
     """An unsupervised learning algorithm that attempts to find discrete groupings within data.
@@ -76,7 +74,7 @@ def __init__(
         half_life_time_size: Optional[int] = None,
         epochs: Optional[int] = None,
         center_factor: Optional[int] = None,
-        eval_metrics: Optional[List[Union[str, PipelineVariable]]] = None,
+        eval_metrics: Optional[List[str]] = None,
         **kwargs
     ):
         """A k-means clustering class :class:`~sagemaker.amazon.AmazonAlgorithmEstimatorBase`.

src/sagemaker/amazon/linear_learner.py

@@ -13,6 +13,7 @@
 """Placeholder docstring"""
 from __future__ import absolute_import
 
+import logging
 from typing import Union, Optional
 
 from sagemaker import image_uris
@@ -28,6 +29,8 @@
 from sagemaker.workflow.entities import PipelineVariable
 from sagemaker.workflow import is_pipeline_variable
 
+logger = logging.getLogger(__name__)
+
 
 class LinearLearner(AmazonAlgorithmEstimatorBase):
     """A supervised learning algorithms used for solving classification or regression problems.
@@ -437,9 +440,14 @@ def _prepare_for_training(self, records, mini_batch_size=None, job_name=None):
         # mini_batch_size can't be greater than number of records or training job fails
         if not mini_batch_size:
             if is_pipeline_variable(self.instance_count):
-                raise ValueError(
-                    "instance_count can not be a pipeline variable when mini_batch_size is not given."
+                logger.warning(
+                    "mini_batch_size is not given in .fit() and instance_count is a "
+                    "pipeline variable (%s) which is only parsed in execution time. "
+                    "Thus setting mini_batch_size to 1, as it can't be greater than "
+                    "number of records per instance_count, otherwise the training job fails.",
+                    type(self.instance_count),
                 )
+                mini_batch_size = 1
             else:
                 mini_batch_size = min(
                     self.DEFAULT_MINI_BATCH_SIZE, max(1, int(num_records / self.instance_count))

src/sagemaker/chainer/estimator.py

@@ -16,7 +16,6 @@
 from typing import Optional, Union, Dict
 
 import logging
-from typing import Union, Optional
 
 from sagemaker.estimator import Framework, EstimatorBase
 from sagemaker.fw_utils import (

src/sagemaker/clarify.py

@@ -37,7 +37,7 @@
 logger = logging.getLogger(__name__)
 
 
-class DataConfig:  #TODO: add PipelineVariable to rest of fields
+class DataConfig:  # TODO: add PipelineVariable to rest of fields
     """Config object related to configurations of the input and output dataset."""
 
     def __init__(
@@ -271,7 +271,7 @@ def get_config(self):
         return copy.deepcopy(self.analysis_config)
 
 
-class ModelConfig: # TODO add pipeline annotation
+class ModelConfig:  # TODO add pipeline annotation
     """Config object related to a model and its endpoint to be created."""
 
     def __init__(

src/sagemaker/estimator.py

@@ -56,7 +56,6 @@
     get_jumpstart_base_name_if_jumpstart_model,
     update_inference_tags_with_jumpstart_training_tags,
 )
-from sagemaker.debugger import RuleBase
 from sagemaker.local import LocalSession
 from sagemaker.model import (
     CONTAINER_LOG_LEVEL_PARAM_NAME,

src/sagemaker/huggingface/estimator.py

@@ -17,7 +17,6 @@
 
 import logging
 import re
-from typing import Optional, Union, Dict
 
 from sagemaker.deprecations import renamed_kwargs
 from sagemaker.estimator import Framework, EstimatorBase
@@ -204,14 +203,8 @@ def __init__(
                     f"Instead got {type(compiler_config)}"
                 )
                 raise ValueError(error_string)
-
-            compiler_config.validate(
-                image_uri=image_uri,
-                instance_type=instance_type,
-                distribution=distribution,
-            )
-
-        self.distribution = distribution or {}
+            if compiler_config:
+                compiler_config.validate(self)
         self.compiler_config = compiler_config
 
     def _validate_args(self, image_uri):

src/sagemaker/inputs.py

@@ -14,6 +14,7 @@
 from __future__ import absolute_import, print_function
 
 from typing import Union, Optional, List
+
 import attr
 
 from sagemaker.workflow.entities import PipelineVariable

src/sagemaker/mxnet/estimator.py

@@ -16,7 +16,6 @@
 from typing import Optional, Dict, Union
 
 import logging
-from typing import Union, Optional, Dict
 
 from packaging.version import Version