test: wait and retry after RESOURCE_EXHAUSTED in tests (#4159)

The ListDatabases RPC is an administrative RPC. This can return a RESOURCE_EXHAUSTED
error if too many administrative RPCs are being executed for the same project in a
short period of time. Back-off and retry if that happens.

Author: olavloite

google-cloud-spanner/src/test/java/com/google/cloud/spanner/IntegrationTestEnv.java

@@ -298,25 +298,44 @@ private void cleanUpOldDatabases(InstanceId instanceId) {
     String TEST_DB_REGEX = "(testdb_(.*)_(.*))|(mysample-(.*))";
 
     logger.log(Level.INFO, "Dropping old test databases from {0}", instanceId.getName());
-    for (Database db : databaseAdminClient.listDatabases(instanceId.getInstance()).iterateAll()) {
+    while (true) {
       try {
-        long timeDiff = currentTimestamp.getSeconds() - db.getCreateTime().getSeconds();
-        // Delete all databases which are more than OLD_DB_THRESHOLD_SECS seconds old.
-        if ((db.getId().getDatabase().matches(TEST_DB_REGEX))
-            && (timeDiff > OLD_DB_THRESHOLD_SECS)) {
-          logger.log(Level.INFO, "Dropping test database {0}", db.getId());
-          if (db.isDropProtectionEnabled()) {
-            Database updatedDatabase =
-                databaseAdminClient.newDatabaseBuilder(db.getId()).disableDropProtection().build();
-            databaseAdminClient
-                .updateDatabase(updatedDatabase, DatabaseField.DROP_PROTECTION)
-                .get();
+        for (Database db :
+            databaseAdminClient.listDatabases(instanceId.getInstance()).iterateAll()) {
+          try {
+            long timeDiff = currentTimestamp.getSeconds() - db.getCreateTime().getSeconds();
+            // Delete all databases which are more than OLD_DB_THRESHOLD_SECS seconds old.
+            if ((db.getId().getDatabase().matches(TEST_DB_REGEX))
+                && (timeDiff > OLD_DB_THRESHOLD_SECS)) {
+              logger.log(Level.INFO, "Dropping test database {0}", db.getId());
+              if (db.isDropProtectionEnabled()) {
+                Database updatedDatabase =
+                    databaseAdminClient
+                        .newDatabaseBuilder(db.getId())
+                        .disableDropProtection()
+                        .build();
+                databaseAdminClient
+                    .updateDatabase(updatedDatabase, DatabaseField.DROP_PROTECTION)
+                    .get();
+              }
+              db.drop();
+              ++numDropped;
+            }
+          } catch (SpannerException | ExecutionException | InterruptedException e) {
+            logger.log(Level.SEVERE, "Failed to drop test database " + db.getId(), e);
           }
-          db.drop();
-          ++numDropped;
         }
-      } catch (SpannerException | ExecutionException | InterruptedException e) {
-        logger.log(Level.SEVERE, "Failed to drop test database " + db.getId(), e);
+        break;
+      } catch (SpannerException exception) {
+        if (exception.getErrorCode() != ErrorCode.RESOURCE_EXHAUSTED) {
+          throw exception;
+        }
+        // Wait a little and try again.
+        try {
+          Thread.sleep(10_000);
+        } catch (InterruptedException interruptedException) {
+          throw SpannerExceptionFactory.propagateInterrupt(interruptedException);
+        }
       }
     }
     logger.log(Level.INFO, "Dropped {0} test database(s)", numDropped);