Add retry logic on HTTP 400 errors for parcel and host functions (#288)

* Add retry logic on HTTP 400 errors for parcel functions
* Add retry logic to HTTP 400 errors for cluster assignment for a host

Signed-off-by: Webster Mudge <[email protected]>

Author: wmudge

plugins/module_utils/parcel_utils.py

@@ -27,6 +27,7 @@
     ParcelResourceApi,
     ParcelsResourceApi,
 )
+from cm_client.rest import ApiException
 
 from ansible_collections.cloudera.cluster.plugins.module_utils.cm_utils import (
     normalize_output,
@@ -89,11 +90,26 @@ def _wait(self, stage: STAGE) -> None:
         return Exception(f"Failed to reach {stage.name}: timeout ({self.timeout} secs)")
 
     def _exec(self, stage: STAGE, func) -> None:
-        func(
-            cluster_name=self.cluster,
-            product=self.product,
-            version=self.version,
-        )
+        retries = 0
+
+        # Retry the function, i.e. start_distribution, if receiving a 400 error due to
+        # potential "eventual consistency" issues with parcel torrents.
+        while True:
+            try:
+                func(
+                    cluster_name=self.cluster,
+                    product=self.product,
+                    version=self.version,
+                )
+                break
+            except ApiException as e:
+                if retries < 4 and e.status == 400:
+                    retries += 1
+                    time.sleep(15)
+                    continue
+                else:
+                    raise e
+
         self._wait(stage)
 
     def remove(self):

plugins/modules/host.py

@@ -339,6 +339,7 @@
       returned: when supported
 """
 
+from time import sleep
 
 from cm_client import (
     ApiHost,
@@ -626,18 +627,30 @@ def process(self):
                             self.diff["after"].update(cluster=cluster.name)
 
                         if not self.module.check_mode:
-                            # Add the host to the cluster
-                            cluster_api.add_hosts(
-                                cluster_name=cluster.name,
-                                body=ApiHostRefList(
-                                    items=[
-                                        ApiHostRef(
-                                            host_id=current.host_id,
-                                            hostname=current.hostname,
-                                        )
-                                    ]
-                                ),
-                            )
+                            # Add the host to the cluster (with simple retry)
+                            add_retry = 0
+
+                            while True:
+                                try:
+                                    cluster_api.add_hosts(
+                                        cluster_name=cluster.name,
+                                        body=ApiHostRefList(
+                                            items=[
+                                                ApiHostRef(
+                                                    host_id=current.host_id,
+                                                    hostname=current.hostname,
+                                                )
+                                            ]
+                                        ),
+                                    )
+                                    break
+                                except ApiException as ae:
+                                    if add_retry < 4 and ae.status == 400:
+                                        add_retry += 1
+                                        sleep(10)
+                                        continue
+                                    else:
+                                        raise ae
 
                             # parcel_api = ParcelResourceApi(self.api_client)
                             # try: