Skip to content

Commit 9604df1

Browse files
DaveCTurnerDaveCTurner
committed
Merge branch 'master' into 2019-09-11-retry-s3-download-on-partial-content
2 parents 22f5703 + fa53ca0 commit 9604df1

File tree

318 files changed

+3522
-3148
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

318 files changed

+3522
-3148
lines changed

‎buildSrc/src/main/java/org/elasticsearch/gradle/EmptyDirTask.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
import org.gradle.api.DefaultTask;
2626
import org.gradle.api.tasks.Input;
2727
import org.gradle.api.tasks.TaskAction;
28-
import org.gradle.internal.nativeintegration.filesystem.Chmod;
28+
import org.gradle.internal.file.Chmod;
2929

3030
/**
3131
* Creates an empty directory.

‎buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java‎

Lines changed: 20 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -139,7 +139,6 @@ public class ElasticsearchNode implements TestClusterConfiguration {
139139
private final Path esStdoutFile;
140140
private final Path esStderrFile;
141141
private final Path tmpDir;
142-
private final Path distroDir;
143142

144143
private int currentDistro = 0;
145144
private TestDistribution testDistribution;
@@ -155,7 +154,6 @@ public class ElasticsearchNode implements TestClusterConfiguration {
155154
this.project = project;
156155
this.reaper = reaper;
157156
workingDir = workingDirBase.toPath().resolve(safeName(name)).toAbsolutePath();
158-
distroDir = workingDir.resolve("distro");
159157
confPathRepo = workingDir.resolve("repo");
160158
configFile = workingDir.resolve("config/elasticsearch.yml");
161159
confPathData = workingDir.resolve("data");
@@ -180,6 +178,10 @@ public Version getVersion() {
180178
return distributions.get(currentDistro).getVersion();
181179
}
182180

181+
public Path getDistroDir() {
182+
return workingDir.resolve("distro").resolve(getVersion() + "-" + testDistribution);
183+
}
184+
183185
@Override
184186
public void setVersion(String version) {
185187
requireNonNull(version, "null version passed when configuring test cluster `" + this + "`");
@@ -507,7 +509,7 @@ private void installModules() {
507509
if (testDistribution == TestDistribution.INTEG_TEST) {
508510
logToProcessStdout("Installing " + modules.size() + "modules");
509511
for (File module : modules) {
510-
Path destination = distroDir.resolve("modules").resolve(module.getName().replace(".zip", "")
512+
Path destination = getDistroDir().resolve("modules").resolve(module.getName().replace(".zip", "")
511513
.replace("-" + getVersion(), "")
512514
.replace("-SNAPSHOT", ""));
513515

@@ -567,16 +569,16 @@ public void user(Map<String, String> userSpec) {
567569

568570
private void runElaticsearchBinScriptWithInput(String input, String tool, String... args) {
569571
if (
570-
Files.exists(distroDir.resolve("bin").resolve(tool)) == false &&
571-
Files.exists(distroDir.resolve("bin").resolve(tool + ".bat")) == false
572+
Files.exists(getDistroDir().resolve("bin").resolve(tool)) == false &&
573+
Files.exists(getDistroDir().resolve("bin").resolve(tool + ".bat")) == false
572574
) {
573575
throw new TestClustersException("Can't run bin script: `" + tool + "` does not exist. " +
574576
"Is this the distribution you expect it to be ?");
575577
}
576578
try (InputStream byteArrayInputStream = new ByteArrayInputStream(input.getBytes(StandardCharsets.UTF_8))) {
577579
LoggedExec.exec(project, spec -> {
578580
spec.setEnvironment(getESEnvironment());
579-
spec.workingDir(distroDir);
581+
spec.workingDir(getDistroDir());
580582
spec.executable(
581583
OS.conditionalString()
582584
.onUnix(() -> "./bin/" + tool)
@@ -631,8 +633,11 @@ private Map<String, String> getESEnvironment() {
631633
})
632634
.collect(Collectors.joining(" "));
633635
}
634-
defaultEnv.put("ES_JAVA_OPTS", "-Xms512m -Xmx512m -ea -esa" +
635-
systemPropertiesString + jvmArgsString
636+
defaultEnv.put("ES_JAVA_OPTS", "-Xms512m -Xmx512m -ea -esa " +
637+
systemPropertiesString + " " +
638+
jvmArgsString + " " +
639+
// Support passing in additional JVM arguments
640+
System.getProperty("tests.jvm.argline", "")
636641
);
637642
defaultEnv.put("ES_TMPDIR", tmpDir.toString());
638643
// Windows requires this as it defaults to `c:\windows` despite ES_TMPDIR
@@ -658,8 +663,8 @@ private void startElasticsearchProcess() {
658663
final ProcessBuilder processBuilder = new ProcessBuilder();
659664

660665
List<String> command = OS.<List<String>>conditional()
661-
.onUnix(() -> Arrays.asList(distroDir.getFileName().resolve("./bin/elasticsearch").toString()))
662-
.onWindows(() -> Arrays.asList("cmd", "/c", distroDir.getFileName().resolve("bin\\elasticsearch.bat").toString()))
666+
.onUnix(() -> Arrays.asList(workingDir.relativize(getDistroDir()).resolve("./bin/elasticsearch").toString()))
667+
.onWindows(() -> Arrays.asList("cmd", "/c", workingDir.relativize(getDistroDir()).resolve("bin\\elasticsearch.bat").toString()))
663668
.supply();
664669
processBuilder.command(command);
665670
processBuilder.directory(workingDir.toFile());
@@ -870,7 +875,9 @@ private void waitForProcessToExit(ProcessHandle processHandle) {
870875
}
871876

872877
private void createWorkingDir(Path distroExtractDir) throws IOException {
873-
syncWithLinks(distroExtractDir, distroDir);
878+
if (Files.exists(getDistroDir()) == false) {
879+
syncWithLinks(distroExtractDir, getDistroDir());
880+
}
874881
// Start configuration from scratch in case of a restart
875882
project.delete(configFile.getParent());
876883
Files.createDirectories(configFile.getParent());
@@ -889,10 +896,7 @@ private void createWorkingDir(Path distroExtractDir) throws IOException {
889896
* @param destinationRoot destination to link to
890897
*/
891898
private void syncWithLinks(Path sourceRoot, Path destinationRoot) {
892-
if (Files.exists(destinationRoot)) {
893-
project.delete(destinationRoot);
894-
}
895-
899+
assert Files.exists(destinationRoot) == false;
896900
try (Stream<Path> stream = Files.walk(sourceRoot)) {
897901
stream.forEach(source -> {
898902
Path relativeDestination = sourceRoot.relativize(source);
@@ -1002,7 +1006,7 @@ private void createConfiguration() {
10021006
);
10031007

10041008
final List<Path> configFiles;
1005-
try (Stream<Path> stream = Files.list(distroDir.resolve("config"))) {
1009+
try (Stream<Path> stream = Files.list(getDistroDir().resolve("config"))) {
10061010
configFiles = stream.collect(Collectors.toList());
10071011
}
10081012
logToProcessStdout("Copying additional config files from distro " + configFiles);

‎buildSrc/src/main/resources/minimumGradleVersion‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
5.5
1+
5.6.2

‎client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -167,20 +167,20 @@
167167
import java.util.concurrent.TimeUnit;
168168
import java.util.stream.Collectors;
169169

170-
import static org.hamcrest.CoreMatchers.equalTo;
171-
import static org.hamcrest.CoreMatchers.hasItem;
172-
import static org.hamcrest.CoreMatchers.hasItems;
173-
import static org.hamcrest.CoreMatchers.not;
174170
import static org.hamcrest.Matchers.allOf;
175171
import static org.hamcrest.Matchers.anyOf;
176172
import static org.hamcrest.Matchers.closeTo;
177173
import static org.hamcrest.Matchers.contains;
178174
import static org.hamcrest.Matchers.containsInAnyOrder;
175+
import static org.hamcrest.Matchers.equalTo;
179176
import static org.hamcrest.Matchers.greaterThan;
180177
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
178+
import static org.hamcrest.Matchers.hasItem;
179+
import static org.hamcrest.Matchers.hasItems;
181180
import static org.hamcrest.Matchers.hasSize;
182181
import static org.hamcrest.Matchers.is;
183182
import static org.hamcrest.Matchers.lessThan;
183+
import static org.hamcrest.Matchers.not;
184184
import static org.hamcrest.Matchers.notNullValue;
185185

186186
public class MachineLearningIT extends ESRestHighLevelClientTestCase {

‎client/rest-high-level/src/test/java/org/elasticsearch/client/ReindexIT.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -292,7 +292,7 @@ public void onFailure(Exception e) {
292292
assertThat(response.getTasks().get(0).getStatus(), instanceOf(RawTaskStatus.class));
293293
assertEquals(Float.toString(requestsPerSecond),
294294
((RawTaskStatus) response.getTasks().get(0).getStatus()).toMap().get("requests_per_second").toString());
295-
taskFinished.await(2, TimeUnit.SECONDS);
295+
assertTrue(taskFinished.await(10, TimeUnit.SECONDS));
296296

297297
// any rethrottling after the update-by-query is done performed with the same taskId should result in a failure
298298
response = execute(new RethrottleRequest(taskIdToRethrottle, requestsPerSecond),
@@ -423,7 +423,7 @@ public void onFailure(Exception e) {
423423
assertThat(response.getTasks().get(0).getStatus(), instanceOf(RawTaskStatus.class));
424424
assertEquals(Float.toString(requestsPerSecond),
425425
((RawTaskStatus) response.getTasks().get(0).getStatus()).toMap().get("requests_per_second").toString());
426-
taskFinished.await(2, TimeUnit.SECONDS);
426+
assertTrue(taskFinished.await(10, TimeUnit.SECONDS));
427427

428428
// any rethrottling after the delete-by-query is done performed with the same taskId should result in a failure
429429
response = execute(new RethrottleRequest(taskIdToRethrottle, requestsPerSecond),

‎client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/DataFrameIndexerPositionTests.java‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -22,31 +22,31 @@
2222
import org.elasticsearch.client.AbstractResponseTestCase;
2323
import org.elasticsearch.common.xcontent.XContentParser;
2424
import org.elasticsearch.common.xcontent.XContentType;
25-
import org.elasticsearch.xpack.core.transform.transforms.DataFrameIndexerPosition;
25+
import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerPosition;
2626

2727
import java.util.LinkedHashMap;
2828
import java.util.Map;
2929

3030
import static org.hamcrest.Matchers.equalTo;
3131

3232
public class DataFrameIndexerPositionTests extends AbstractResponseTestCase<
33-
DataFrameIndexerPosition,
33+
TransformIndexerPosition,
3434
org.elasticsearch.client.transform.transforms.DataFrameIndexerPosition> {
3535

36-
public static DataFrameIndexerPosition fromHlrc(
36+
public static TransformIndexerPosition fromHlrc(
3737
org.elasticsearch.client.transform.transforms.DataFrameIndexerPosition instance) {
3838
if (instance == null) {
3939
return null;
4040
}
41-
return new DataFrameIndexerPosition(instance.getIndexerPosition(), instance.getBucketsPosition());
41+
return new TransformIndexerPosition(instance.getIndexerPosition(), instance.getBucketsPosition());
4242
}
4343

44-
public static DataFrameIndexerPosition randomDataFrameIndexerPosition() {
45-
return new DataFrameIndexerPosition(randomPositionMap(), randomPositionMap());
44+
public static TransformIndexerPosition randomDataFrameIndexerPosition() {
45+
return new TransformIndexerPosition(randomPositionMap(), randomPositionMap());
4646
}
4747

4848
@Override
49-
protected DataFrameIndexerPosition createServerTestInstance(XContentType xContentType) {
49+
protected TransformIndexerPosition createServerTestInstance(XContentType xContentType) {
5050
return randomDataFrameIndexerPosition();
5151
}
5252

@@ -56,7 +56,7 @@ protected org.elasticsearch.client.transform.transforms.DataFrameIndexerPosition
5656
}
5757

5858
@Override
59-
protected void assertInstances(DataFrameIndexerPosition serverTestInstance,
59+
protected void assertInstances(TransformIndexerPosition serverTestInstance,
6060
org.elasticsearch.client.transform.transforms.DataFrameIndexerPosition clientInstance) {
6161
assertThat(serverTestInstance.getIndexerPosition(), equalTo(clientInstance.getIndexerPosition()));
6262
assertThat(serverTestInstance.getBucketsPosition(), equalTo(clientInstance.getBucketsPosition()));

‎client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/DataFrameIndexerTransformStatsTests.java‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -21,17 +21,17 @@
2121

2222
import org.elasticsearch.client.AbstractHlrcXContentTestCase;
2323
import org.elasticsearch.common.xcontent.XContentParser;
24-
import org.elasticsearch.xpack.core.transform.transforms.DataFrameIndexerTransformStats;
24+
import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats;
2525

2626
import java.io.IOException;
2727

2828
public class DataFrameIndexerTransformStatsTests extends AbstractHlrcXContentTestCase<
29-
DataFrameIndexerTransformStats,
29+
TransformIndexerStats,
3030
org.elasticsearch.client.transform.transforms.DataFrameIndexerTransformStats> {
3131

32-
public static DataFrameIndexerTransformStats fromHlrc(
32+
public static TransformIndexerStats fromHlrc(
3333
org.elasticsearch.client.transform.transforms.DataFrameIndexerTransformStats instance) {
34-
return new DataFrameIndexerTransformStats(
34+
return new TransformIndexerStats(
3535
instance.getNumPages(),
3636
instance.getNumDocuments(),
3737
instance.getOutputDocuments(),
@@ -54,13 +54,13 @@ public org.elasticsearch.client.transform.transforms.DataFrameIndexerTransformSt
5454
}
5555

5656
@Override
57-
public DataFrameIndexerTransformStats convertHlrcToInternal(
57+
public TransformIndexerStats convertHlrcToInternal(
5858
org.elasticsearch.client.transform.transforms.DataFrameIndexerTransformStats instance) {
5959
return fromHlrc(instance);
6060
}
6161

62-
public static DataFrameIndexerTransformStats randomStats() {
63-
return new DataFrameIndexerTransformStats(randomLongBetween(10L, 10000L),
62+
public static TransformIndexerStats randomStats() {
63+
return new TransformIndexerStats(randomLongBetween(10L, 10000L),
6464
randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L),
6565
randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L),
6666
randomLongBetween(0L, 10000L),
@@ -70,13 +70,13 @@ public static DataFrameIndexerTransformStats randomStats() {
7070
}
7171

7272
@Override
73-
protected DataFrameIndexerTransformStats createTestInstance() {
73+
protected TransformIndexerStats createTestInstance() {
7474
return randomStats();
7575
}
7676

7777
@Override
78-
protected DataFrameIndexerTransformStats doParseInstance(XContentParser parser) throws IOException {
79-
return DataFrameIndexerTransformStats.fromXContent(parser);
78+
protected TransformIndexerStats doParseInstance(XContentParser parser) throws IOException {
79+
return TransformIndexerStats.fromXContent(parser);
8080
}
8181

8282
@Override

‎client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/DataFrameTransformCheckpointStatsTests.java‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -22,34 +22,34 @@
2222
import org.elasticsearch.client.AbstractResponseTestCase;
2323
import org.elasticsearch.common.xcontent.XContentParser;
2424
import org.elasticsearch.common.xcontent.XContentType;
25-
import org.elasticsearch.xpack.core.transform.transforms.DataFrameTransformCheckpointStats;
25+
import org.elasticsearch.xpack.core.transform.transforms.TransformCheckpointStats;
2626

2727
import java.io.IOException;
2828

2929
import static org.hamcrest.Matchers.equalTo;
3030

3131
public class DataFrameTransformCheckpointStatsTests extends AbstractResponseTestCase<
32-
DataFrameTransformCheckpointStats,
32+
TransformCheckpointStats,
3333
org.elasticsearch.client.transform.transforms.DataFrameTransformCheckpointStats> {
3434

35-
public static DataFrameTransformCheckpointStats fromHlrc(
35+
public static TransformCheckpointStats fromHlrc(
3636
org.elasticsearch.client.transform.transforms.DataFrameTransformCheckpointStats instance) {
37-
return new DataFrameTransformCheckpointStats(instance.getCheckpoint(),
37+
return new TransformCheckpointStats(instance.getCheckpoint(),
3838
DataFrameIndexerPositionTests.fromHlrc(instance.getPosition()),
3939
DataFrameTransformProgressTests.fromHlrc(instance.getCheckpointProgress()),
4040
instance.getTimestampMillis(),
4141
instance.getTimeUpperBoundMillis());
4242
}
4343

44-
public static DataFrameTransformCheckpointStats randomDataFrameTransformCheckpointStats() {
45-
return new DataFrameTransformCheckpointStats(randomLongBetween(1, 1_000_000),
44+
public static TransformCheckpointStats randomDataFrameTransformCheckpointStats() {
45+
return new TransformCheckpointStats(randomLongBetween(1, 1_000_000),
4646
DataFrameIndexerPositionTests.randomDataFrameIndexerPosition(),
4747
randomBoolean() ? null : DataFrameTransformProgressTests.randomDataFrameTransformProgress(),
4848
randomLongBetween(1, 1_000_000), randomLongBetween(0, 1_000_000));
4949
}
5050

5151
@Override
52-
protected DataFrameTransformCheckpointStats createServerTestInstance(XContentType xContentType) {
52+
protected TransformCheckpointStats createServerTestInstance(XContentType xContentType) {
5353
return randomDataFrameTransformCheckpointStats();
5454
}
5555

@@ -60,7 +60,7 @@ protected org.elasticsearch.client.transform.transforms.DataFrameTransformCheckp
6060
}
6161

6262
@Override
63-
protected void assertInstances(DataFrameTransformCheckpointStats serverTestInstance,
63+
protected void assertInstances(TransformCheckpointStats serverTestInstance,
6464
org.elasticsearch.client.transform.transforms.DataFrameTransformCheckpointStats clientInstance) {
6565
assertThat(serverTestInstance.getCheckpoint(), equalTo(clientInstance.getCheckpoint()));
6666
assertThat(serverTestInstance.getPosition().getBucketsPosition(), equalTo(clientInstance.getPosition().getBucketsPosition()));

‎client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/DataFrameTransformCheckpointingInfoTests.java‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -21,19 +21,19 @@
2121

2222
import org.elasticsearch.client.AbstractHlrcXContentTestCase;
2323
import org.elasticsearch.common.xcontent.XContentParser;
24-
import org.elasticsearch.xpack.core.transform.transforms.DataFrameTransformCheckpointingInfo;
24+
import org.elasticsearch.xpack.core.transform.transforms.TransformCheckpointingInfo;
2525

2626
import java.io.IOException;
2727
import java.time.Instant;
2828
import java.util.function.Predicate;
2929

3030
public class DataFrameTransformCheckpointingInfoTests extends AbstractHlrcXContentTestCase<
31-
DataFrameTransformCheckpointingInfo,
31+
TransformCheckpointingInfo,
3232
org.elasticsearch.client.transform.transforms.DataFrameTransformCheckpointingInfo> {
3333

34-
public static DataFrameTransformCheckpointingInfo fromHlrc(
34+
public static TransformCheckpointingInfo fromHlrc(
3535
org.elasticsearch.client.transform.transforms.DataFrameTransformCheckpointingInfo instance) {
36-
return new DataFrameTransformCheckpointingInfo(
36+
return new TransformCheckpointingInfo(
3737
DataFrameTransformCheckpointStatsTests.fromHlrc(instance.getLast()),
3838
DataFrameTransformCheckpointStatsTests.fromHlrc(instance.getNext()),
3939
instance.getOperationsBehind(),
@@ -46,27 +46,27 @@ public org.elasticsearch.client.transform.transforms.DataFrameTransformCheckpoin
4646
}
4747

4848
@Override
49-
public DataFrameTransformCheckpointingInfo convertHlrcToInternal(
49+
public TransformCheckpointingInfo convertHlrcToInternal(
5050
org.elasticsearch.client.transform.transforms.DataFrameTransformCheckpointingInfo instance) {
5151
return fromHlrc(instance);
5252
}
5353

54-
public static DataFrameTransformCheckpointingInfo randomDataFrameTransformCheckpointingInfo() {
55-
return new DataFrameTransformCheckpointingInfo(
54+
public static TransformCheckpointingInfo randomDataFrameTransformCheckpointingInfo() {
55+
return new TransformCheckpointingInfo(
5656
DataFrameTransformCheckpointStatsTests.randomDataFrameTransformCheckpointStats(),
5757
DataFrameTransformCheckpointStatsTests.randomDataFrameTransformCheckpointStats(),
5858
randomNonNegativeLong(),
5959
randomBoolean() ? null : Instant.ofEpochMilli(randomNonNegativeLong()));
6060
}
6161

6262
@Override
63-
protected DataFrameTransformCheckpointingInfo createTestInstance() {
63+
protected TransformCheckpointingInfo createTestInstance() {
6464
return randomDataFrameTransformCheckpointingInfo();
6565
}
6666

6767
@Override
68-
protected DataFrameTransformCheckpointingInfo doParseInstance(XContentParser parser) throws IOException {
69-
return DataFrameTransformCheckpointingInfo.fromXContent(parser);
68+
protected TransformCheckpointingInfo doParseInstance(XContentParser parser) throws IOException {
69+
return TransformCheckpointingInfo.fromXContent(parser);
7070
}
7171

7272
@Override

0 commit comments

Comments
 (0)