Skip to content

Commit 08abbdf

Browse files
authored
Use fixture to test repository-url module (#29355)
This commit adds a YAML integration test for the repository-url module that uses a fixture to test URL based repositories on both http:// and file:// prefixes.
1 parent 25d411e commit 08abbdf

File tree

5 files changed

+510
-18
lines changed

5 files changed

+510
-18
lines changed

modules/repository-url/build.gradle

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,36 @@
1616
* specific language governing permissions and limitations
1717
* under the License.
1818
*/
19+
import org.elasticsearch.gradle.test.AntFixture
1920

2021
esplugin {
2122
description 'Module for URL repository'
2223
classname 'org.elasticsearch.plugin.repository.url.URLRepositoryPlugin'
2324
}
2425

26+
forbiddenApisTest {
27+
// we are using jdk-internal instead of jdk-non-portable to allow for com.sun.net.httpserver.* usage
28+
bundledSignatures -= 'jdk-non-portable'
29+
bundledSignatures += 'jdk-internal'
30+
}
31+
32+
// This directory is shared between two URL repositories and one FS repository in YAML integration tests
33+
File repositoryDir = new File(project.buildDir, "shared-repository")
34+
35+
/** A task to start the URLFixture which exposes the repositoryDir over HTTP **/
36+
task urlFixture(type: AntFixture) {
37+
doFirst {
38+
repositoryDir.mkdirs()
39+
}
40+
env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }"
41+
executable = new File(project.runtimeJavaHome, 'bin/java')
42+
args 'org.elasticsearch.repositories.url.URLFixture', baseDir, "${repositoryDir.absolutePath}"
43+
}
44+
2545
integTestCluster {
26-
setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
46+
dependsOn urlFixture
47+
// repositoryDir is used by a FS repository to create snapshots
48+
setting 'path.repo', "${repositoryDir.absolutePath}"
49+
// repositoryDir is used by two URL repositories to restore snapshots
50+
setting 'repositories.url.allowed_urls', "http://snapshot.test*,http://${ -> urlFixture.addressAndPort }"
2751
}

modules/repository-url/src/test/java/org/elasticsearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java

Lines changed: 84 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,31 @@
2121

2222
import com.carrotsearch.randomizedtesting.annotations.Name;
2323
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
24-
24+
import org.apache.http.HttpEntity;
25+
import org.apache.http.entity.ContentType;
26+
import org.apache.http.nio.entity.NStringEntity;
27+
import org.elasticsearch.client.Response;
28+
import org.elasticsearch.common.Strings;
29+
import org.elasticsearch.common.settings.Settings;
30+
import org.elasticsearch.common.xcontent.ToXContent;
31+
import org.elasticsearch.common.xcontent.XContentBuilder;
32+
import org.elasticsearch.common.xcontent.support.XContentMapValues;
33+
import org.elasticsearch.repositories.fs.FsRepository;
34+
import org.elasticsearch.rest.RestStatus;
2535
import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
2636
import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
37+
import org.junit.Before;
38+
39+
import java.io.IOException;
40+
import java.net.InetAddress;
41+
import java.net.URL;
42+
import java.util.List;
43+
import java.util.Map;
44+
45+
import static java.util.Collections.emptyMap;
46+
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
47+
import static org.hamcrest.Matchers.equalTo;
48+
import static org.hamcrest.Matchers.hasSize;
2749

2850
public class RepositoryURLClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
2951

@@ -35,5 +57,66 @@ public RepositoryURLClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate
3557
public static Iterable<Object[]> parameters() throws Exception {
3658
return ESClientYamlSuiteTestCase.createParameters();
3759
}
60+
61+
/**
62+
* This method registers 3 snapshot/restore repositories:
63+
* - repository-fs: this FS repository is used to create snapshots.
64+
* - repository-url: this URL repository is used to restore snapshots created using the previous repository. It uses
65+
* the URLFixture to restore snapshots over HTTP.
66+
* - repository-file: similar as the previous repository but using a file:// prefix instead of http://.
67+
**/
68+
@Before
69+
public void registerRepositories() throws IOException {
70+
Response clusterSettingsResponse = client().performRequest("GET", "/_cluster/settings?include_defaults=true" +
71+
"&filter_path=defaults.path.repo,defaults.repositories.url.allowed_urls");
72+
Map<String, Object> clusterSettings = entityAsMap(clusterSettingsResponse);
73+
74+
@SuppressWarnings("unchecked")
75+
List<String> pathRepo = (List<String>) XContentMapValues.extractValue("defaults.path.repo", clusterSettings);
76+
assertThat(pathRepo, hasSize(1));
77+
78+
// Create a FS repository using the path.repo location
79+
Response createFsRepositoryResponse = client().performRequest("PUT", "_snapshot/repository-fs", emptyMap(),
80+
buildRepositorySettings(FsRepository.TYPE, Settings.builder().put("location", pathRepo.get(0)).build()));
81+
assertThat(createFsRepositoryResponse.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus()));
82+
83+
// Create a URL repository using the file://{path.repo} URL
84+
Response createFileRepositoryResponse = client().performRequest("PUT", "_snapshot/repository-file", emptyMap(),
85+
buildRepositorySettings(URLRepository.TYPE, Settings.builder().put("url", "file://" + pathRepo.get(0)).build()));
86+
assertThat(createFileRepositoryResponse.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus()));
87+
88+
// Create a URL repository using the http://{fixture} URL
89+
@SuppressWarnings("unchecked")
90+
List<String> allowedUrls = (List<String>) XContentMapValues.extractValue("defaults.repositories.url.allowed_urls", clusterSettings);
91+
for (String allowedUrl : allowedUrls) {
92+
try {
93+
InetAddress inetAddress = InetAddress.getByName(new URL(allowedUrl).getHost());
94+
if (inetAddress.isAnyLocalAddress() || inetAddress.isLoopbackAddress()) {
95+
Response createUrlRepositoryResponse = client().performRequest("PUT", "_snapshot/repository-url", emptyMap(),
96+
buildRepositorySettings(URLRepository.TYPE, Settings.builder().put("url", allowedUrl).build()));
97+
assertThat(createUrlRepositoryResponse.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus()));
98+
break;
99+
}
100+
} catch (Exception e) {
101+
logger.debug("Failed to resolve inet address for allowed URL [{}], skipping", allowedUrl);
102+
}
103+
}
104+
}
105+
106+
private static HttpEntity buildRepositorySettings(final String type, final Settings settings) throws IOException {
107+
try (XContentBuilder builder = jsonBuilder()) {
108+
builder.startObject();
109+
{
110+
builder.field("type", type);
111+
builder.startObject("settings");
112+
{
113+
settings.toXContent(builder, ToXContent.EMPTY_PARAMS);
114+
}
115+
builder.endObject();
116+
}
117+
builder.endObject();
118+
return new NStringEntity(Strings.toString(builder), ContentType.APPLICATION_JSON);
119+
}
120+
}
38121
}
39122

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.elasticsearch.repositories.url;
20+
21+
import com.sun.net.httpserver.HttpExchange;
22+
import com.sun.net.httpserver.HttpHandler;
23+
import com.sun.net.httpserver.HttpServer;
24+
import org.elasticsearch.common.SuppressForbidden;
25+
import org.elasticsearch.mocksocket.MockHttpServer;
26+
import org.elasticsearch.rest.RestStatus;
27+
28+
import java.io.IOException;
29+
import java.lang.management.ManagementFactory;
30+
import java.net.Inet6Address;
31+
import java.net.InetAddress;
32+
import java.net.InetSocketAddress;
33+
import java.net.SocketAddress;
34+
import java.nio.charset.StandardCharsets;
35+
import java.nio.file.Files;
36+
import java.nio.file.Path;
37+
import java.nio.file.Paths;
38+
import java.nio.file.StandardCopyOption;
39+
import java.util.Map;
40+
import java.util.Objects;
41+
42+
import static java.util.Collections.emptyMap;
43+
import static java.util.Collections.singleton;
44+
import static java.util.Collections.singletonMap;
45+
46+
/**
47+
* This {@link URLFixture} exposes a filesystem directory over HTTP. It is used in repository-url
48+
* integration tests to expose a directory created by a regular FS repository.
49+
*/
50+
public class URLFixture {
51+
52+
public static void main(String[] args) throws Exception {
53+
if (args == null || args.length != 2) {
54+
throw new IllegalArgumentException("URLFixture <working directory> <repository directory>");
55+
}
56+
57+
final InetSocketAddress socketAddress = new InetSocketAddress(InetAddress.getLoopbackAddress(), 0);
58+
final HttpServer httpServer = MockHttpServer.createHttp(socketAddress, 0);
59+
60+
try {
61+
final Path workingDirectory = dir(args[0]);
62+
/// Writes the PID of the current Java process in a `pid` file located in the working directory
63+
writeFile(workingDirectory, "pid", ManagementFactory.getRuntimeMXBean().getName().split("@")[0]);
64+
65+
final String addressAndPort = addressToString(httpServer.getAddress());
66+
// Writes the address and port of the http server in a `ports` file located in the working directory
67+
writeFile(workingDirectory, "ports", addressAndPort);
68+
69+
// Exposes the repository over HTTP
70+
final String url = "http://" + addressAndPort;
71+
httpServer.createContext("/", new ResponseHandler(dir(args[1])));
72+
httpServer.start();
73+
74+
// Wait to be killed
75+
Thread.sleep(Long.MAX_VALUE);
76+
77+
} finally {
78+
httpServer.stop(0);
79+
}
80+
}
81+
82+
@SuppressForbidden(reason = "Paths#get is fine - we don't have environment here")
83+
private static Path dir(final String dir) {
84+
return Paths.get(dir);
85+
}
86+
87+
private static void writeFile(final Path dir, final String fileName, final String content) throws IOException {
88+
final Path tempPidFile = Files.createTempFile(dir, null, null);
89+
Files.write(tempPidFile, singleton(content));
90+
Files.move(tempPidFile, dir.resolve(fileName), StandardCopyOption.ATOMIC_MOVE);
91+
}
92+
93+
private static String addressToString(final SocketAddress address) {
94+
final InetSocketAddress inetSocketAddress = (InetSocketAddress) address;
95+
if (inetSocketAddress.getAddress() instanceof Inet6Address) {
96+
return "[" + inetSocketAddress.getHostString() + "]:" + inetSocketAddress.getPort();
97+
} else {
98+
return inetSocketAddress.getHostString() + ":" + inetSocketAddress.getPort();
99+
}
100+
}
101+
102+
static class ResponseHandler implements HttpHandler {
103+
104+
private final Path repositoryDir;
105+
106+
ResponseHandler(final Path repositoryDir) {
107+
this.repositoryDir = repositoryDir;
108+
}
109+
110+
@Override
111+
public void handle(HttpExchange exchange) throws IOException {
112+
Response response;
113+
if ("GET".equalsIgnoreCase(exchange.getRequestMethod())) {
114+
String path = exchange.getRequestURI().toString();
115+
if (path.length() > 0 && path.charAt(0) == '/') {
116+
path = path.substring(1);
117+
}
118+
119+
Path normalizedRepositoryDir = repositoryDir.normalize();
120+
Path normalizedPath = normalizedRepositoryDir.resolve(path).normalize();
121+
122+
if (normalizedPath.startsWith(normalizedRepositoryDir)) {
123+
if (Files.exists(normalizedPath) && Files.isReadable(normalizedPath) && Files.isRegularFile(normalizedPath)) {
124+
byte[] content = Files.readAllBytes(normalizedPath);
125+
Map<String, String> headers = singletonMap("Content-Length", String.valueOf(content.length));
126+
response = new Response(RestStatus.OK, headers, "application/octet-stream", content);
127+
} else {
128+
response = new Response(RestStatus.NOT_FOUND, emptyMap(), "text/plain", new byte[0]);
129+
}
130+
} else {
131+
response = new Response(RestStatus.FORBIDDEN, emptyMap(), "text/plain", new byte[0]);
132+
}
133+
} else {
134+
response = new Response(RestStatus.INTERNAL_SERVER_ERROR, emptyMap(), "text/plain",
135+
"Unsupported HTTP method".getBytes(StandardCharsets.UTF_8));
136+
}
137+
exchange.sendResponseHeaders(response.status.getStatus(), response.body.length);
138+
if (response.body.length > 0) {
139+
exchange.getResponseBody().write(response.body);
140+
}
141+
exchange.close();
142+
}
143+
}
144+
145+
/**
146+
* Represents a HTTP Response.
147+
*/
148+
static class Response {
149+
150+
final RestStatus status;
151+
final Map<String, String> headers;
152+
final String contentType;
153+
final byte[] body;
154+
155+
Response(final RestStatus status, final Map<String, String> headers, final String contentType, final byte[] body) {
156+
this.status = Objects.requireNonNull(status);
157+
this.headers = Objects.requireNonNull(headers);
158+
this.contentType = Objects.requireNonNull(contentType);
159+
this.body = Objects.requireNonNull(body);
160+
}
161+
}
162+
}

0 commit comments

Comments
 (0)