Move repository-gcs fixture test to QA project and allow testing against external service

plugins/repository-gcs/build.gradle

@@ -1,8 +1,3 @@
-import org.elasticsearch.gradle.test.AntFixture
-
-import java.security.KeyPair
-import java.security.KeyPairGenerator
-
 /*
  * Licensed to Elasticsearch under one or more contributor
  * license agreements. See the NOTICE file distributed with
@@ -58,50 +53,7 @@ thirdPartyAudit.excludes = [
         'org.apache.log.Logger',
 ]
 
-forbiddenApisTest {
-  // we are using jdk-internal instead of jdk-non-portable to allow for com.sun.net.httpserver.* usage
-  bundledSignatures -= 'jdk-non-portable'
-  bundledSignatures += 'jdk-internal'
-}
-
-/** A task to start the GoogleCloudStorageFixture which emulates a Google Cloud Storage service **/
-task googleCloudStorageFixture(type: AntFixture) {
-    dependsOn compileTestJava
-    env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }"
-    executable = new File(project.runtimeJavaHome, 'bin/java')
-    args 'org.elasticsearch.repositories.gcs.GoogleCloudStorageFixture', baseDir, 'bucket_test'
-}
-
-/** A service account file that points to the Google Cloud Storage service emulated by the fixture **/
-File serviceAccountFile = new File(project.buildDir, "generated-resources/service_account_test.json")
-task createServiceAccountFile() {
-  dependsOn googleCloudStorageFixture
-  doLast {
-    KeyPairGenerator keyPairGenerator = KeyPairGenerator.getInstance("RSA")
-    keyPairGenerator.initialize(1024)
-    KeyPair keyPair = keyPairGenerator.generateKeyPair()
-    String encodedKey = Base64.getEncoder().encodeToString(keyPair.private.getEncoded())
-
-    serviceAccountFile.parentFile.mkdirs()
-    serviceAccountFile.setText("{\n" +
-            '  "type": "service_account",\n' +
-            '  "project_id": "integration_test",\n' +
-            '  "private_key_id": "' + UUID.randomUUID().toString() + '",\n' +
-            '  "private_key": "-----BEGIN PRIVATE KEY-----\\n' + encodedKey + '\\n-----END PRIVATE KEY-----\\n",\n' +
-            '  "client_email": "[email protected]",\n' +
-            '  "client_id": "123456789101112130594",\n' +
-            "  \"auth_uri\": \"http://${googleCloudStorageFixture.addressAndPort}/o/oauth2/auth\",\n" +
-            "  \"token_uri\": \"http://${googleCloudStorageFixture.addressAndPort}/o/oauth2/token\",\n" +
-            '  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",\n' +
-            '  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/integration_test%40appspot.gserviceaccount.com"\n' +
-            '}', 'UTF-8')
-  }
-}
-
-integTestCluster {
-  dependsOn createServiceAccountFile, googleCloudStorageFixture
-  keystoreFile 'gcs.client.integration_test.credentials_file', "${serviceAccountFile.absolutePath}"
-
-  /* Use a closure on the string to delay evaluation until tests are executed */
-  setting 'gcs.client.integration_test.endpoint', "http://${ -> googleCloudStorageFixture.addressAndPort }"
-}
+check {
+  // also execute the QA tests when testing the plugin
+  dependsOn 'qa:google-cloud-storage:check'
+}

plugins/repository-gcs/qa/google-cloud-storage/build.gradle

@@ -0,0 +1,115 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.elasticsearch.gradle.MavenFilteringHack
+import org.elasticsearch.gradle.test.AntFixture
+
+import java.security.KeyPair
+import java.security.KeyPairGenerator
+
+apply plugin: 'elasticsearch.standalone-rest-test'
+apply plugin: 'elasticsearch.rest-test'
+
+dependencies {
+    testCompile project(path: ':plugins:repository-gcs', configuration: 'runtime')
+}
+
+integTestCluster {
+    plugin ':plugins:repository-gcs'
+}
+
+forbiddenApisTest {
+    // we are using jdk-internal instead of jdk-non-portable to allow for com.sun.net.httpserver.* usage
+    bundledSignatures -= 'jdk-non-portable'
+    bundledSignatures += 'jdk-internal'
+}
+
+boolean useFixture = false
+
+String gcsServiceAccount = System.getenv("google_storage_service_account")
+String gcsBucket = System.getenv("google_storage_bucket")
+String gcsBasePath = System.getenv("google_storage_base_path")
+
+File serviceAccountFile = null
+if (!gcsServiceAccount && !gcsBucket && !gcsBasePath) {
+    serviceAccountFile = new File(project.buildDir, 'generated-resources/service_account_test.json')
+    gcsBucket = 'bucket_test'
+    gcsBasePath = 'integration_test'
+    useFixture = true
+} else {
+    serviceAccountFile = new File(gcsServiceAccount)
+    if (serviceAccountFile.exists() == false || serviceAccountFile.canRead() == false) {
+        throw new FileNotFoundException(gcsServiceAccount, "Google Storage service account file does not exist or is not readable")
+    }
+}
+
+/** A task to start the GoogleCloudStorageFixture which emulates a Google Cloud Storage service **/
+task googleCloudStorageFixture(type: AntFixture) {
+    dependsOn compileTestJava
+    env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }"
+    executable = new File(project.runtimeJavaHome, 'bin/java')
+    args 'org.elasticsearch.repositories.gcs.GoogleCloudStorageFixture', baseDir, 'bucket_test'
+}
+
+/** A service account file that points to the Google Cloud Storage service emulated by the fixture **/
+task createServiceAccountFile() {
+    dependsOn googleCloudStorageFixture
+    doLast {
+        KeyPairGenerator keyPairGenerator = KeyPairGenerator.getInstance("RSA")
+        keyPairGenerator.initialize(1024)
+        KeyPair keyPair = keyPairGenerator.generateKeyPair()
+        String encodedKey = Base64.getEncoder().encodeToString(keyPair.private.getEncoded())
+
+        serviceAccountFile.parentFile.mkdirs()
+        serviceAccountFile.setText("{\n" +
+                '  "type": "service_account",\n' +
+                '  "project_id": "integration_test",\n' +
+                '  "private_key_id": "' + UUID.randomUUID().toString() + '",\n' +
+                '  "private_key": "-----BEGIN PRIVATE KEY-----\\n' + encodedKey + '\\n-----END PRIVATE KEY-----\\n",\n' +
+                '  "client_email": "[email protected]",\n' +
+                '  "client_id": "123456789101112130594",\n' +
+                "  \"auth_uri\": \"http://${googleCloudStorageFixture.addressAndPort}/o/oauth2/auth\",\n" +
+                "  \"token_uri\": \"http://${googleCloudStorageFixture.addressAndPort}/o/oauth2/token\",\n" +
+                '  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",\n' +
+                '  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/integration_test%40appspot.gserviceaccount.com"\n' +
+                '}', 'UTF-8')
+    }
+}
+
+Map<String, Object> expansions = [
+        'bucket': gcsBucket,
+        'base_path': gcsBasePath
+]
+
+processTestResources {
+    inputs.properties(expansions)
+    MavenFilteringHack.filter(it, expansions)
+}
+
+integTestCluster {
+    keystoreFile 'gcs.client.integration_test.credentials_file', "${serviceAccountFile.absolutePath}"
+
+    if (useFixture) {
+        dependsOn createServiceAccountFile, googleCloudStorageFixture
+        /* Use a closure on the string to delay evaluation until tests are executed */
+        setting 'gcs.client.integration_test.endpoint', "http://${ -> googleCloudStorageFixture.addressAndPort }"
+    } else {
+        println "Using an external service to test the repository-gcs plugin"
+    }
+}

plugins/repository-gcs/qa/google-cloud-storage/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepositoryClientYamlTestSuiteIT.java

@@ -0,0 +1,37 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.gcs;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+
+public class GoogleCloudStorageRepositoryClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
+
+    public GoogleCloudStorageRepositoryClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return createParameters();
+    }
+}

plugins/repository-gcs/qa/google-cloud-storage/src/test/resources/rest-api-spec/test/repository_gcs/10_repository.yml

@@ -0,0 +1,177 @@
+# Integration tests for repository-gcs
+---
+"Snapshot/Restore with repository-gcs":
+
+  # Register repository
+  - do:
+      snapshot.create_repository:
+        repository: repository
+        body:
+          type: gcs
+          settings:
+            bucket: ${bucket}
+            client: "integration_test"
+            base_path: ${base_path}
+
+  - match: { acknowledged: true }
+
+  # Get repository
+  - do:
+      snapshot.get_repository:
+        repository: repository
+
+  - match: { repository.settings.bucket : ${bucket} }
+  - match: { repository.settings.client : "integration_test" }
+  - match: { repository.settings.base_path : ${base_path} }
+
+  # Index documents
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    1
+          - snapshot: one
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    2
+          - snapshot: one
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    3
+          - snapshot: one
+
+  - do:
+      count:
+        index: docs
+
+  - match: {count: 3}
+
+  # Create a first snapshot
+  - do:
+      snapshot.create:
+        repository: repository
+        snapshot: snapshot-one
+        wait_for_completion: true
+
+  - match: { snapshot.snapshot: snapshot-one }
+  - match: { snapshot.state : SUCCESS }
+  - match: { snapshot.include_global_state: true }
+  - match: { snapshot.shards.failed : 0 }
+
+  - do:
+      snapshot.status:
+        repository: repository
+        snapshot: snapshot-one
+
+  - is_true: snapshots
+  - match: { snapshots.0.snapshot: snapshot-one }
+  - match: { snapshots.0.state : SUCCESS }
+
+  # Index more documents
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    4
+          - snapshot: two
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    5
+          - snapshot: two
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    6
+          - snapshot: two
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    7
+          - snapshot: two
+
+  - do:
+      count:
+        index: docs
+
+  - match: {count: 7}
+
+  # Create a second snapshot
+  - do:
+      snapshot.create:
+        repository: repository
+        snapshot: snapshot-two
+        wait_for_completion: true
+
+  - match: { snapshot.snapshot: snapshot-two }
+  - match: { snapshot.state : SUCCESS }
+  - match: { snapshot.shards.failed : 0 }
+
+  - do:
+      snapshot.get:
+        repository: repository
+        snapshot: snapshot-one,snapshot-two
+
+  - is_true: snapshots
+  - match: { snapshots.0.state : SUCCESS }
+  - match: { snapshots.1.state : SUCCESS }
+
+  # Delete the index
+  - do:
+      indices.delete:
+        index: docs
+
+  # Restore the second snapshot
+  - do:
+      snapshot.restore:
+        repository: repository
+        snapshot: snapshot-two
+        wait_for_completion: true
+
+  - do:
+      count:
+        index: docs
+
+  - match: {count: 7}
+
+  # Delete the index again
+  - do:
+      indices.delete:
+        index: docs
+
+  # Restore the first snapshot
+  - do:
+      snapshot.restore:
+        repository: repository
+        snapshot: snapshot-one
+        wait_for_completion: true
+
+  - do:
+      count:
+        index: docs
+
+  - match: {count: 3}
+
+  # Remove the snapshots
+  - do:
+      snapshot.delete:
+        repository: repository
+        snapshot: snapshot-two
+
+  - do:
+      snapshot.delete:
+        repository: repository
+        snapshot: snapshot-one
+
+  # Remove our repository
+  - do:
+     snapshot.delete_repository:
+       repository: repository