change: Clean up test resources to remove model files (aws#756)

* change: Clean up test resources to remove model files

* fix: Change experiment enums to upper case

* change: Upgrade boto3 and update test to validate mixed case name

* fix: Update as per latest botocore release and backend change

Co-authored-by: Dewen Qi <[email protected]>

Author: qidewenwhen

.gitignore

@@ -32,4 +32,4 @@ env/
 .python-version
 **/_repack_script_launcher.sh
 tests/data/**/_repack_model.py
-tests/data/experiment/resources/sagemaker-beta-1.0.tar.gz
+tests/data/experiment/sagemaker-dev-1.0.tar.gz

setup.py

@@ -48,7 +48,7 @@ def read_requirements(filename):
 # Declare minimal set for installation
 required_packages = [
     "attrs>=20.3.0,<23",
-    "boto3>=1.26.20,<2.0",
+    "boto3>=1.26.28,<2.0",
     "google-pasta",
     "numpy>=1.9.0,<2.0",
     "protobuf>=3.1,<4.0",

src/sagemaker/experiments/_metrics.py

@@ -155,9 +155,7 @@ def __init__(self, metric_name, value, timestamp=None, step=None):
 
         self.MetricName = metric_name
         self.Value = float(value)
-        # Update timestamp to milliseconds
-        # to be compatible with the metrics service
-        self.Timestamp = timestamp * 1000
+        self.Timestamp = timestamp
         if step is not None:
             if not isinstance(step, int):
                 raise ValueError("step must be int.")
@@ -169,10 +167,12 @@ def to_record(self):
 
     def to_raw_metric_data(self):
         """Converts the metric data to a BatchPutMetrics RawMetricData item"""
+        # Convert timestamp from float to timestamp str.
+        # Otherwise will get ParamValidationError
         raw_metric_data = {
             "MetricName": self.MetricName,
             "Value": self.Value,
-            "Timestamp": int(self.Timestamp),
+            "Timestamp": str(int(self.Timestamp)),
         }
         if self.Step is not None:
             raw_metric_data["Step"] = int(self.Step)

src/sagemaker/experiments/run.py

@@ -61,12 +61,9 @@
 TRIAL_NAME_TEMPLATE = "Default-Run-Group-{}"
 MAX_RUN_TC_ARTIFACTS_LEN = 30
 MAX_NAME_LEN_IN_BACKEND = 120
-UNKNOWN_NAME = "unknown"
-EXP_NAME_BASE = "Sagemaker-Experiment"
 EXPERIMENT_NAME = "ExperimentName"
 TRIAL_NAME = "TrialName"
 RUN_NAME = "RunName"
-TRIAL_COMPONENT_NAME = "TrialComponentName"
 DELIMITER = "-"
 RUN_TC_TAG_KEY = "sagemaker:trial-component-source"
 RUN_TC_TAG_VALUE = "run"
@@ -76,15 +73,15 @@
 class SortByType(Enum):
     """The type of property by which to sort the `list_runs` results."""
 
-    CreationTime = "CreationTime"
-    Name = "Name"
+    CREATION_TIME = "CreationTime"
+    NAME = "Name"
 
 
 class SortOrderType(Enum):
     """The type of order to sort the list or search results."""
 
-    Ascending = "Ascending"
-    Descending = "Descending"
+    ASCENDING = "Ascending"
+    DESCENDING = "Descending"
 
 
 class Run(object):
@@ -169,9 +166,9 @@ def __init__(
             sagemaker_session=sagemaker_session,
         )
         if is_existed:
-            logger.warning(
+            logger.info(
                 "The Run (%s) under experiment (%s) already exists. Loading it. "
-                "Note, sagemaker.experiments.load_run is recommended to use when "
+                "Note: sagemaker.experiments.load_run is recommended to use when "
                 "the desired Run already exists.",
                 self.run_name,
                 self.experiment_name,
@@ -496,10 +493,10 @@ def close(self):
 
     @staticmethod
     def _generate_trial_name(base_name) -> str:
-        """Generate the reserved trial name based on run name
+        """Generate the reserved trial name based on experiment name
 
         Args:
-            base_name (str): The run_name of this `Run` object.
+            base_name (str): The ``experiment_name`` of this ``Run`` object.
         """
         available_length = MAX_NAME_LEN_IN_BACKEND - len(TRIAL_NAME_TEMPLATE)
         return TRIAL_NAME_TEMPLATE.format(base_name[:available_length])
@@ -760,8 +757,8 @@ def list_runs(
     sagemaker_session: Optional["Session"] = None,
     max_results: Optional[int] = None,
     next_token: Optional[str] = None,
-    sort_by: SortByType = SortByType.CreationTime,
-    sort_order: SortOrderType = SortOrderType.Descending,
+    sort_by: SortByType = SortByType.CREATION_TIME,
+    sort_order: SortOrderType = SortOrderType.DESCENDING,
 ) -> list:
     """Return a list of `Run` objects matching the given criteria.
 
@@ -776,11 +773,11 @@ def list_runs(
             manages interactions with Amazon SageMaker APIs and any other
             AWS services needed. If not specified, one is created using the
             default AWS configuration chain.
-        max_results (int): maximum number of Run objects to retrieve (default: None).
-        next_token (str): token for next page of results (default: None).
-        sort_by (SortByType): Which property to sort results by. One of Name, CreationTime
-            (default: CreationTime).
-        sort_order (SortOrderType): One of Ascending, or Descending (default: Descending).
+        max_results (int): Maximum number of Run objects to retrieve (default: None).
+        next_token (str): Token for next page of results (default: None).
+        sort_by (SortByType): The property to sort results by. One of NAME, CREATION_TIME
+            (default: CREATION_TIME).
+        sort_order (SortOrderType): One of ASCENDING, or DESCENDING (default: DESCENDING).
 
     Returns:
         list: A list of `Run` objects.

tests/__init__.py

@@ -11,19 +11,3 @@
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
 from __future__ import absolute_import
-
-import os
-
-
-# TODO-experiment-plus: Remove this line, which loads the internal boto models.
-# The corresponding model jsons were generated from the coral model package and should
-# be updated regularly.
-normal_json = "file://./tests/data/experiment/resources/sagemaker-2017-07-24.normal.json"
-os.system(f"aws configure add-model --service-model {normal_json} --service-name sagemaker")
-
-public_metrics_model_json = (
-    "file://./tests/data/experiment/resources/sagemaker-metrics-2022-09-30.normal.json"
-)
-os.system(
-    f"aws configure add-model --service-model {public_metrics_model_json} --service-name sagemaker-metrics"
-)

tests/data/experiment/docker/Dockerfile

@@ -10,7 +10,6 @@
 #  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
 #  express or implied. See the License for the specific language governing
 #  permissions and limitations under the License.
-import importlib
 import logging
 import os
 import pickle as pkl
@@ -19,22 +18,10 @@
 import numpy as np
 import sagemaker_xgboost_container.encoder as xgb_encoders
 
+sdk_name = "sagemaker-dev-1.0.tar.gz"
+code_dir = "/opt/ml/code"
 
-sm_json = "sagemaker-2017-07-24.normal.json"
-metrics_json = "sagemaker-metrics-2022-09-30.normal.json"
-beta_sdk = "sagemaker-beta-1.0.tar.gz"
-resource_dir = "/opt/ml/code/resources"
-
-os.system("pip install awscli")
-os.system(
-    f"aws configure add-model --service-model file://{resource_dir}/{sm_json} --service-name sagemaker"
-)
-os.system(
-    f"aws configure add-model --service-model file://{resource_dir}/{metrics_json} --service-name sagemaker-metrics"
-)
-importlib.reload(boto3)  # Reload boto3 to let the added API models take effect
-
-sdk_file = f"{resource_dir}/{beta_sdk}"
+sdk_file = f"{code_dir}/{sdk_name}"
 os.system(f"pip install {sdk_file}")
 
 from sagemaker.session import Session

tests/data/experiment/docker/scripts/train_job_script_for_run_clz.py

@@ -13,25 +13,11 @@
 """This script file runs on SageMaker processing job"""
 from __future__ import absolute_import
 
-import importlib
 import logging
 import os
 import boto3
 
-sm_json = "sagemaker-2017-07-24.normal.json"
-metrics_json = "sagemaker-metrics-2022-09-30.normal.json"
-beta_sdk = "sagemaker-beta-1.0.tar.gz"
-resource_dir = "resources"
-
-os.system(
-    f"aws configure add-model --service-model file://{resource_dir}/{sm_json} --service-name sagemaker"
-)
-os.system(
-    f"aws configure add-model --service-model file://{resource_dir}/{metrics_json} --service-name sagemaker-metrics"
-)
-importlib.reload(boto3)  # Reload boto3 to let the added API models take effect
-
-sdk_file = f"{resource_dir}/{beta_sdk}"
+sdk_file = "sagemaker-dev-1.0.tar.gz"
 os.system(f"pip install {sdk_file}")