Merge pull request #581 from saurabh3949/master

Fix ray checkpointing issues

Author: vrkhare

reinforcement_learning/rl_roboschool_ray/common/sagemaker_rl/configuration_list.py

@@ -2,7 +2,7 @@
 
 
 class ConfigurationList(object):
-    """Helper Object for converting CLI arguments (or SageMaker hyperparameters) 
+    """Helper Object for converting CLI arguments (or SageMaker hyperparameters)
     into Coach configuration.
     """
 
@@ -65,6 +65,8 @@ def _set_rl_property_value(self, obj, key, val, path=""):
     def _autotype(self, val):
         """Converts string to an int or float as possible.
         """
+        if type(val) == bool:
+            return val
         try:
             return int(val)
         except ValueError:
@@ -83,7 +85,7 @@ def _parse_type(self, key, val):
         Automatically detects ints and floats when possible.
         If the key takes the form "foo:bar" then it looks in ALLOWED_TYPES
         for an entry of bar, and instantiates one of those objects, passing
-        val to the constructor.  So if key="foo:EnvironmentSteps" then 
+        val to the constructor.  So if key="foo:EnvironmentSteps" then
         """
         val = self._autotype(val)
         if key.find(":") > 0:
@@ -93,5 +95,3 @@ def _parse_type(self, key, val):
                 raise ValueError("Unrecognized object type %s.  Allowed values are %s" % (obj_type, self.ALLOWED_TYPES.keys()))
             val = cls(val)
         return key, val
-
-

reinforcement_learning/rl_roboschool_ray/common/sagemaker_rl/ray_launcher.py

@@ -98,6 +98,8 @@ def customize_experiment_config(self, config):
         # Set output dir to intermediate
         # TODO: move this to before customer-specified so they can override
         hyperparams_dict["rl.training.local_dir"] = INTERMEDIATE_DIR
+        hyperparams_dict["rl.training.checkpoint_at_end"] = True
+        hyperparams_dict["rl.training.checkpoint_freq"] = 10
         self.hyperparameters = ConfigurationList()  # TODO: move to shared
         for name, value in hyperparams_dict.items():
             # self.map_hyperparameter(name, val) #TODO
@@ -106,10 +108,7 @@ def customize_experiment_config(self, config):
                 self.hyperparameters.store(name, value)
                 #             else:
                 #                 raise ValueError("Unknown hyperparameter %s" % name)
-
         self.hyperparameters.apply_subset(config, "rl.")
-        hyperparams_dict["rl.training.checkpoint_at_end"] = True
-        hyperparams_dict["rl.training.checkpoint_freq"] = 10
         return config
 
     def get_all_host_names(self):

reinforcement_learning/rl_roboschool_ray/rl_roboschool_ray.ipynb

@@ -306,7 +306,6 @@
     "\n",
     "s3_url = \"s3://{}/{}\".format(s3_bucket,job_name)\n",
     "\n",
-    "\n",
     "intermediate_folder_key = \"{}/output/intermediate/\".format(job_name)\n",
     "intermediate_url = \"s3://{}/{}\".format(s3_bucket, intermediate_folder_key)\n",
     "\n",
@@ -333,9 +332,9 @@
    "outputs": [],
    "source": [
     "recent_videos = wait_for_s3_object(\n",
-    "        s3_bucket, intermediate_folder_key, tmp_dir, \n",
-    "        fetch_only=(lambda obj: obj.key.endswith(\".mp4\") and obj.size>0), \n",
-    "        limit=10, training_job_name=job_name)"
+    "            s3_bucket, intermediate_folder_key, tmp_dir, \n",
+    "            fetch_only=(lambda obj: obj.key.endswith(\".mp4\") and obj.size>0), \n",
+    "            limit=10, training_job_name=job_name)"
    ]
   },
   {
@@ -366,14 +365,17 @@
     "%matplotlib inline\n",
     "from sagemaker.analytics import TrainingJobAnalytics\n",
     "\n",
-    "df = TrainingJobAnalytics(job_name, ['episode_reward_mean']).dataframe()\n",
-    "num_metrics = len(df)\n",
-    "if num_metrics == 0:\n",
-    "    print(\"No algorithm metrics found in CloudWatch\")\n",
+    "if not local_mode:\n",
+    "    df = TrainingJobAnalytics(job_name, ['episode_reward_mean']).dataframe()\n",
+    "    num_metrics = len(df)\n",
+    "    if num_metrics == 0:\n",
+    "        print(\"No algorithm metrics found in CloudWatch\")\n",
+    "    else:\n",
+    "        plt = df.plot(x='timestamp', y='value', figsize=(12,5), legend=True, style='b-')\n",
+    "        plt.set_ylabel('Mean reward per episode')\n",
+    "        plt.set_xlabel('Training time (s)')\n",
     "else:\n",
-    "    plt = df.plot(x='timestamp', y='value', figsize=(12,5), legend=True, style='b-')\n",
-    "    plt.set_ylabel('Mean reward per episode')\n",
-    "    plt.set_xlabel('Training time (s)')"
+    "    print(\"Can't plot metrics in local mode.\")"
    ]
   },
   {
@@ -403,7 +405,11 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "model_tar_key = \"{}/output/model.tar.gz\".format(job_name)\n",
+    "if local_mode:\n",
+    "    model_tar_key = \"{}/model.tar.gz\".format(job_name)\n",
+    "else:\n",
+    "    model_tar_key = \"{}/output/model.tar.gz\".format(job_name)\n",
+    "    \n",
     "local_checkpoint_dir = \"{}/model\".format(tmp_dir)\n",
     "\n",
     "wait_for_s3_object(s3_bucket, model_tar_key, tmp_dir, training_job_name=job_name)  \n",