Skip to content

Commit 44bf784

Browse files
original-brownbearoriginal-brownbear
authored
Add Infrastructure to Run 3rd Party Repository Tests (#42586) (#42604)
* Add Infrastructure to Run 3rd Party Repository Tests * Add infrastructure to run third party repository tests using our standard JUnit infrastructure * This is a prerequisite of #42189
1 parent 635ce0c commit 44bf784

File tree

7 files changed

+373
-5
lines changed

7 files changed

+373
-5
lines changed

‎plugins/repository-azure/build.gradle‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,3 +71,24 @@ testClusters {
7171
keystore 'azure.client.integration_test.key', 'azure_key'
7272
}
7373
}
74+
75+
String azureAccount = System.getenv("azure_storage_account")
76+
String azureKey = System.getenv("azure_storage_key")
77+
String azureContainer = System.getenv("azure_storage_container")
78+
String azureBasePath = System.getenv("azure_storage_base_path")
79+
80+
test {
81+
exclude '**/AzureStorageCleanupThirdPartyTests.class'
82+
}
83+
84+
task thirdPartyTest(type: Test) {
85+
include '**/AzureStorageCleanupThirdPartyTests.class'
86+
systemProperty 'test.azure.account', azureAccount ? azureAccount : ""
87+
systemProperty 'test.azure.key', azureKey ? azureKey : ""
88+
systemProperty 'test.azure.container', azureContainer ? azureContainer : ""
89+
systemProperty 'test.azure.base', azureBasePath ? azureBasePath : ""
90+
}
91+
92+
if (azureAccount || azureKey || azureContainer || azureBasePath) {
93+
check.dependsOn(thirdPartyTest)
94+
}

‎plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.repositories.azure;
21+
22+
import org.elasticsearch.action.support.master.AcknowledgedResponse;
23+
import org.elasticsearch.common.settings.MockSecureSettings;
24+
import org.elasticsearch.common.settings.SecureSettings;
25+
import org.elasticsearch.common.settings.Settings;
26+
import org.elasticsearch.plugins.Plugin;
27+
import org.elasticsearch.repositories.AbstractThirdPartyRepositoryTestCase;
28+
29+
import java.util.Collection;
30+
31+
import static org.hamcrest.Matchers.blankOrNullString;
32+
import static org.hamcrest.Matchers.equalTo;
33+
import static org.hamcrest.Matchers.not;
34+
35+
public class AzureStorageCleanupThirdPartyTests extends AbstractThirdPartyRepositoryTestCase {
36+
37+
@Override
38+
protected Collection<Class<? extends Plugin>> getPlugins() {
39+
return pluginList(AzureRepositoryPlugin.class);
40+
}
41+
42+
@Override
43+
protected SecureSettings credentials() {
44+
assertThat(System.getProperty("test.azure.account"), not(blankOrNullString()));
45+
assertThat(System.getProperty("test.azure.key"), not(blankOrNullString()));
46+
assertThat(System.getProperty("test.azure.container"), not(blankOrNullString()));
47+
assertThat(System.getProperty("test.azure.base"), not(blankOrNullString()));
48+
49+
MockSecureSettings secureSettings = new MockSecureSettings();
50+
secureSettings.setString("azure.client.default.account", System.getProperty("test.azure.account"));
51+
secureSettings.setString("azure.client.default.key", System.getProperty("test.azure.key"));
52+
return secureSettings;
53+
}
54+
55+
@Override
56+
protected void createRepository(String repoName) {
57+
AcknowledgedResponse putRepositoryResponse = client().admin().cluster().preparePutRepository(repoName)
58+
.setType("azure")
59+
.setSettings(Settings.builder()
60+
.put("container", System.getProperty("test.azure.container"))
61+
.put("base_path", System.getProperty("test.azure.base"))
62+
).get();
63+
assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
64+
}
65+
}

‎plugins/repository-gcs/build.gradle‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
import java.nio.file.Files
2+
13
/*
24
* Licensed to Elasticsearch under one or more contributor
35
* license agreements. See the NOTICE file distributed with
@@ -122,3 +124,22 @@ check {
122124
// also execute the QA tests when testing the plugin
123125
dependsOn 'qa:google-cloud-storage:check'
124126
}
127+
128+
String gcsServiceAccount = System.getenv("google_storage_service_account")
129+
String gcsBucket = System.getenv("google_storage_bucket")
130+
String gcsBasePath = System.getenv("google_storage_base_path")
131+
132+
test {
133+
exclude '**/GoogleCloudStorageThirdPartyTests.class'
134+
}
135+
136+
task thirdPartyTest(type: Test) {
137+
include '**/GoogleCloudStorageThirdPartyTests.class'
138+
systemProperty 'test.google.account', gcsServiceAccount ? Base64.encoder.encodeToString(Files.readAllBytes(file(gcsServiceAccount).toPath())) : ""
139+
systemProperty 'test.google.bucket', gcsBucket ? gcsBucket : ""
140+
systemProperty 'test.google.base', gcsBasePath ? gcsBasePath : "/"
141+
}
142+
143+
if (gcsServiceAccount || gcsBucket || gcsBasePath) {
144+
check.dependsOn(thirdPartyTest)
145+
}

‎plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageThirdPartyTests.java‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.repositories.gcs;
21+
22+
import org.elasticsearch.action.support.master.AcknowledgedResponse;
23+
import org.elasticsearch.common.settings.MockSecureSettings;
24+
import org.elasticsearch.common.settings.SecureSettings;
25+
import org.elasticsearch.common.settings.Settings;
26+
import org.elasticsearch.plugins.Plugin;
27+
import org.elasticsearch.repositories.AbstractThirdPartyRepositoryTestCase;
28+
29+
import java.util.Base64;
30+
import java.util.Collection;
31+
32+
import static org.hamcrest.Matchers.blankOrNullString;
33+
import static org.hamcrest.Matchers.equalTo;
34+
import static org.hamcrest.Matchers.not;
35+
36+
public class GoogleCloudStorageThirdPartyTests extends AbstractThirdPartyRepositoryTestCase {
37+
38+
@Override
39+
protected Collection<Class<? extends Plugin>> getPlugins() {
40+
return pluginList(GoogleCloudStoragePlugin.class);
41+
}
42+
43+
@Override
44+
protected SecureSettings credentials() {
45+
assertThat(System.getProperty("test.google.account"), not(blankOrNullString()));
46+
assertThat(System.getProperty("test.google.bucket"), not(blankOrNullString()));
47+
48+
MockSecureSettings secureSettings = new MockSecureSettings();
49+
secureSettings.setFile("gcs.client.default.credentials_file",
50+
Base64.getDecoder().decode(System.getProperty("test.google.account")));
51+
return secureSettings;
52+
}
53+
54+
@Override
55+
protected void createRepository(final String repoName) {
56+
AcknowledgedResponse putRepositoryResponse = client().admin().cluster().preparePutRepository("test-repo")
57+
.setType("gcs")
58+
.setSettings(Settings.builder()
59+
.put("bucket", System.getProperty("test.google.bucket"))
60+
.put("base_path", System.getProperty("test.google.base", "/"))
61+
).get();
62+
assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
63+
}
64+
}

‎plugins/repository-s3/build.gradle‎

Lines changed: 38 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ test {
7575
// these are tested explicitly in separate test tasks
7676
exclude '**/*CredentialsTests.class'
7777
exclude '**/S3BlobStoreRepositoryTests.class'
78+
exclude '**/S3RepositoryThirdPartyTests.class'
7879
}
7980

8081
boolean useFixture = false
@@ -134,6 +135,14 @@ if (!s3EC2Bucket && !s3EC2BasePath && !s3ECSBucket && !s3ECSBasePath) {
134135
throw new IllegalArgumentException("not all options specified to run EC2/ECS tests are present")
135136
}
136137

138+
task thirdPartyTest(type: Test) {
139+
include '**/S3RepositoryThirdPartyTests.class'
140+
systemProperty 'test.s3.account', s3PermanentAccessKey
141+
systemProperty 'test.s3.key', s3PermanentSecretKey
142+
systemProperty 'test.s3.bucket', s3PermanentBucket
143+
systemProperty 'test.s3.base', s3PermanentBasePath
144+
}
145+
137146
if (useFixture) {
138147
apply plugin: 'elasticsearch.test.fixtures'
139148
task writeDockerFile {
@@ -151,6 +160,32 @@ if (useFixture) {
151160
dependsOn(writeDockerFile)
152161
}
153162

163+
def minioAddress = {
164+
int minioPort = postProcessFixture.ext."test.fixtures.minio-fixture.tcp.9000"
165+
assert minioPort > 0
166+
return 'http://127.0.0.1:' + minioPort
167+
}
168+
169+
File minioAddressFile = new File(project.buildDir, 'generated-resources/s3Fixture.address')
170+
171+
// We can't lazy evaluate a system property for the Minio address passed to JUnit so we write it to a resource file
172+
// and pass its name instead.
173+
task writeMinioAddress {
174+
dependsOn tasks.bundlePlugin, tasks.postProcessFixture
175+
outputs.file(minioAddressFile)
176+
doLast {
177+
file(minioAddressFile).text = "${ -> minioAddress.call() }"
178+
}
179+
}
180+
181+
thirdPartyTest {
182+
dependsOn writeMinioAddress
183+
inputs.file(minioAddressFile)
184+
systemProperty 'test.s3.endpoint', minioAddressFile.name
185+
}
186+
187+
BuildPlugin.requireDocker(tasks.thirdPartyTest)
188+
154189
task integTestMinio(type: RestIntegTestTask) {
155190
description = "Runs REST tests using the Minio repository."
156191
dependsOn tasks.bundlePlugin, tasks.postProcessFixture
@@ -169,11 +204,7 @@ if (useFixture) {
169204
testClusters.integTestMinio {
170205
keystore 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey
171206
keystore 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey
172-
setting 's3.client.integration_test_permanent.endpoint', {
173-
int minioPort = postProcessFixture.ext."test.fixtures.minio-fixture.tcp.9000"
174-
assert minioPort > 0
175-
return 'http://127.0.0.1:' + minioPort
176-
}
207+
setting 's3.client.integration_test_permanent.endpoint', minioAddress
177208
plugin file(tasks.bundlePlugin.archiveFile)
178209
}
179210

@@ -191,6 +222,8 @@ if (useFixture) {
191222
}
192223
}
193224

225+
check.dependsOn(thirdPartyTest)
226+
194227
File parentFixtures = new File(project.buildDir, "fixtures")
195228
File s3FixtureFile = new File(parentFixtures, 's3Fixture.properties')
196229

‎plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.elasticsearch.repositories.s3;
20+
21+
import org.elasticsearch.action.support.master.AcknowledgedResponse;
22+
import org.elasticsearch.common.settings.MockSecureSettings;
23+
import org.elasticsearch.common.settings.SecureSettings;
24+
import org.elasticsearch.common.settings.Settings;
25+
import org.elasticsearch.plugins.Plugin;
26+
import org.elasticsearch.repositories.AbstractThirdPartyRepositoryTestCase;
27+
import org.elasticsearch.test.StreamsUtils;
28+
29+
import java.io.IOException;
30+
import java.util.Collection;
31+
32+
import static org.hamcrest.Matchers.blankOrNullString;
33+
import static org.hamcrest.Matchers.equalTo;
34+
import static org.hamcrest.Matchers.not;
35+
36+
public class S3RepositoryThirdPartyTests extends AbstractThirdPartyRepositoryTestCase {
37+
38+
@Override
39+
protected Collection<Class<? extends Plugin>> getPlugins() {
40+
return pluginList(S3RepositoryPlugin.class);
41+
}
42+
43+
@Override
44+
protected SecureSettings credentials() {
45+
assertThat(System.getProperty("test.s3.account"), not(blankOrNullString()));
46+
assertThat(System.getProperty("test.s3.key"), not(blankOrNullString()));
47+
assertThat(System.getProperty("test.s3.bucket"), not(blankOrNullString()));
48+
49+
MockSecureSettings secureSettings = new MockSecureSettings();
50+
secureSettings.setString("s3.client.default.access_key", System.getProperty("test.s3.account"));
51+
secureSettings.setString("s3.client.default.secret_key", System.getProperty("test.s3.key"));
52+
return secureSettings;
53+
}
54+
55+
@Override
56+
protected void createRepository(String repoName) {
57+
Settings.Builder settings = Settings.builder()
58+
.put("bucket", System.getProperty("test.s3.bucket"))
59+
.put("base_path", System.getProperty("test.s3.base", "/"));
60+
final String endpointPath = System.getProperty("test.s3.endpoint");
61+
if (endpointPath != null) {
62+
try {
63+
settings = settings.put("endpoint", StreamsUtils.copyToStringFromClasspath("/" + endpointPath));
64+
} catch (IOException e) {
65+
throw new AssertionError(e);
66+
}
67+
}
68+
AcknowledgedResponse putRepositoryResponse = client().admin().cluster().preparePutRepository("test-repo")
69+
.setType("s3")
70+
.setSettings(settings).get();
71+
assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
72+
}
73+
}

0 commit comments

Comments
 (0)