SmokeTestWatcherWithSecurityIT: Retry if failures searching (#43781)

Retry if failures searching .watcher-history

Catch exceptions and fail (which allows the assertBusy to retry)
if an error happens from the search against .watcher-history.

Related: #42409
Related: #39169

Author: jakelandis

x-pack/qa/smoke-test-watcher-with-security/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherWithSecurityIT.java

@@ -8,6 +8,7 @@
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
@@ -330,6 +331,10 @@ private ObjectPath getWatchHistoryEntry(String watchId, String state) throws Exc
                 String watchid = objectPath.evaluate("hits.hits.0._source.watch_id");
                 assertThat(watchid, is(watchId));
                 objectPathReference.set(objectPath);
+            } catch (ResponseException e) {
+                final String err = "Failed to perform search of watcher history";
+                logger.info(err, e);
+                fail(err);
             }
         });
         return objectPathReference.get();

x-pack/qa/smoke-test-watcher/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherTestSuiteIT.java

@@ -216,8 +216,8 @@ private ObjectPath getWatchHistoryEntry(String watchId) throws Exception {
                 assertThat("watch_id for hit 0 in watcher history", foundWatchId, is(watchId));
                 objectPathReference.set(objectPath);
             } catch (ResponseException e) {
-                final String err = "Failed to perform search of watcher history - " + e;
-                logger.info(err);
+                final String err = "Failed to perform search of watcher history";
+                logger.info(err, e);
                 fail(err);
             }
         });