Skip to content

Commit d3b05a1

Browse files
original-brownbearoriginal-brownbear
committed
Merge remote-tracking branch 'elastic/master' into retry-sending-snapshot-shard-status-updates
2 parents 3378d3d + 44d5ad9 commit d3b05a1

File tree

84 files changed

+1276
-687
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

84 files changed

+1276
-687
lines changed

‎.ci/bwcVersions‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,4 +16,5 @@ BWC_VERSION:
1616
- "7.5.2"
1717
- "7.5.3"
1818
- "7.6.0"
19+
- "7.7.0"
1920
- "8.0.0"

‎buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -426,7 +426,7 @@ class BuildPlugin implements Plugin<Project> {
426426
dependencyNode.appendNode('groupId', dependency.group)
427427
dependencyNode.appendNode('artifactId', dependency.getDependencyProject().convention.getPlugin(BasePluginConvention).archivesBaseName)
428428
dependencyNode.appendNode('version', dependency.version)
429-
dependencyNode.appendNode('scope', 'runtime')
429+
dependencyNode.appendNode('scope', 'compile')
430430
}
431431
}
432432
}

‎buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/RestTestRunnerTask.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,22 @@
11
package org.elasticsearch.gradle.testclusters;
22

3+
import org.elasticsearch.gradle.tool.Boilerplate;
4+
import org.gradle.api.provider.Provider;
5+
import org.gradle.api.services.internal.BuildServiceRegistryInternal;
36
import org.gradle.api.tasks.CacheableTask;
7+
import org.gradle.api.tasks.Internal;
48
import org.gradle.api.tasks.Nested;
59
import org.gradle.api.tasks.testing.Test;
10+
import org.gradle.internal.resources.ResourceLock;
11+
import org.gradle.internal.resources.SharedResource;
612

13+
import java.util.ArrayList;
714
import java.util.Collection;
815
import java.util.Collections;
916
import java.util.HashSet;
17+
import java.util.List;
18+
19+
import static org.elasticsearch.gradle.testclusters.TestClustersPlugin.THROTTLE_SERVICE_NAME;
1020

1121
/**
1222
* Customized version of Gradle {@link Test} task which tracks a collection of {@link ElasticsearchCluster} as a task input. We must do this
@@ -47,4 +57,19 @@ public Collection<ElasticsearchCluster> getClusters() {
4757
return clusters;
4858
}
4959

60+
@Override
61+
@Internal
62+
public List<ResourceLock> getSharedResources() {
63+
List<ResourceLock> locks = new ArrayList<>(super.getSharedResources());
64+
BuildServiceRegistryInternal serviceRegistry = getServices().get(BuildServiceRegistryInternal.class);
65+
Provider<TestClustersThrottle> throttleProvider = Boilerplate.getBuildService(serviceRegistry, THROTTLE_SERVICE_NAME);
66+
SharedResource resource = serviceRegistry.forService(throttleProvider);
67+
68+
int nodeCount = clusters.stream().mapToInt(cluster -> cluster.getNodes().size()).sum();
69+
if (nodeCount > 0) {
70+
locks.add(resource.getResourceLock(Math.min(nodeCount, resource.getMaxUsages())));
71+
}
72+
73+
return Collections.unmodifiableList(locks);
74+
}
5075
}

‎buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java‎

Lines changed: 88 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import org.elasticsearch.gradle.DistributionDownloadPlugin;
2222
import org.elasticsearch.gradle.ReaperPlugin;
2323
import org.elasticsearch.gradle.ReaperService;
24+
import org.elasticsearch.gradle.tool.Boilerplate;
2425
import org.gradle.api.NamedDomainObjectContainer;
2526
import org.gradle.api.Plugin;
2627
import org.gradle.api.Project;
@@ -30,53 +31,50 @@
3031
import org.gradle.api.invocation.Gradle;
3132
import org.gradle.api.logging.Logger;
3233
import org.gradle.api.logging.Logging;
34+
import org.gradle.api.provider.Provider;
3335
import org.gradle.api.tasks.TaskState;
3436

3537
import java.io.File;
3638

3739
public class TestClustersPlugin implements Plugin<Project> {
3840

39-
private static final String LIST_TASK_NAME = "listTestClusters";
4041
public static final String EXTENSION_NAME = "testClusters";
41-
private static final String REGISTRY_EXTENSION_NAME = "testClustersRegistry";
42+
public static final String THROTTLE_SERVICE_NAME = "testClustersThrottle";
4243

44+
private static final String LIST_TASK_NAME = "listTestClusters";
45+
private static final String REGISTRY_SERVICE_NAME = "testClustersRegistry";
4346
private static final Logger logger = Logging.getLogger(TestClustersPlugin.class);
4447

45-
private ReaperService reaper;
46-
4748
@Override
4849
public void apply(Project project) {
4950
project.getPlugins().apply(DistributionDownloadPlugin.class);
50-
5151
project.getRootProject().getPluginManager().apply(ReaperPlugin.class);
52-
reaper = project.getRootProject().getExtensions().getByType(ReaperService.class);
52+
53+
ReaperService reaper = project.getRootProject().getExtensions().getByType(ReaperService.class);
5354

5455
// enable the DSL to describe clusters
55-
NamedDomainObjectContainer<ElasticsearchCluster> container = createTestClustersContainerExtension(project);
56+
NamedDomainObjectContainer<ElasticsearchCluster> container = createTestClustersContainerExtension(project, reaper);
5657

5758
// provide a task to be able to list defined clusters.
5859
createListClustersTask(project, container);
5960

60-
if (project.getRootProject().getExtensions().findByName(REGISTRY_EXTENSION_NAME) == null) {
61-
TestClustersRegistry registry = project.getRootProject()
62-
.getExtensions()
63-
.create(REGISTRY_EXTENSION_NAME, TestClustersRegistry.class);
64-
65-
// When we know what tasks will run, we claim the clusters of those task to differentiate between clusters
66-
// that are defined in the build script and the ones that will actually be used in this invocation of gradle
67-
// we use this information to determine when the last task that required the cluster executed so that we can
68-
// terminate the cluster right away and free up resources.
69-
configureClaimClustersHook(project.getGradle(), registry);
61+
// register cluster registry as a global build service
62+
project.getGradle().getSharedServices().registerIfAbsent(REGISTRY_SERVICE_NAME, TestClustersRegistry.class, spec -> {});
7063

71-
// Before each task, we determine if a cluster needs to be started for that task.
72-
configureStartClustersHook(project.getGradle(), registry);
64+
// register throttle so we only run at most max-workers/2 nodes concurrently
65+
project.getGradle()
66+
.getSharedServices()
67+
.registerIfAbsent(
68+
THROTTLE_SERVICE_NAME,
69+
TestClustersThrottle.class,
70+
spec -> spec.getMaxParallelUsages().set(project.getGradle().getStartParameter().getMaxWorkerCount() / 2)
71+
);
7372

74-
// After each task we determine if there are clusters that are no longer needed.
75-
configureStopClustersHook(project.getGradle(), registry);
76-
}
73+
// register cluster hooks
74+
project.getRootProject().getPluginManager().apply(TestClustersHookPlugin.class);
7775
}
7876

79-
private NamedDomainObjectContainer<ElasticsearchCluster> createTestClustersContainerExtension(Project project) {
77+
private NamedDomainObjectContainer<ElasticsearchCluster> createTestClustersContainerExtension(Project project, ReaperService reaper) {
8078
// Create an extensions that allows describing clusters
8179
NamedDomainObjectContainer<ElasticsearchCluster> container = project.container(
8280
ElasticsearchCluster.class,
@@ -95,52 +93,78 @@ private void createListClustersTask(Project project, NamedDomainObjectContainer<
9593
);
9694
}
9795

98-
private static void configureClaimClustersHook(Gradle gradle, TestClustersRegistry registry) {
99-
// Once we know all the tasks that need to execute, we claim all the clusters that belong to those and count the
100-
// claims so we'll know when it's safe to stop them.
101-
gradle.getTaskGraph().whenReady(taskExecutionGraph -> {
102-
taskExecutionGraph.getAllTasks()
103-
.stream()
104-
.filter(task -> task instanceof TestClustersAware)
105-
.map(task -> (TestClustersAware) task)
106-
.flatMap(task -> task.getClusters().stream())
107-
.forEach(registry::claimCluster);
108-
});
109-
}
96+
static class TestClustersHookPlugin implements Plugin<Project> {
97+
@Override
98+
public void apply(Project project) {
99+
if (project != project.getRootProject()) {
100+
throw new IllegalStateException(this.getClass().getName() + " can only be applied to the root project.");
101+
}
102+
103+
Provider<TestClustersRegistry> registryProvider = Boilerplate.getBuildService(
104+
project.getGradle().getSharedServices(),
105+
REGISTRY_SERVICE_NAME
106+
);
107+
TestClustersRegistry registry = registryProvider.get();
108+
109+
// When we know what tasks will run, we claim the clusters of those task to differentiate between clusters
110+
// that are defined in the build script and the ones that will actually be used in this invocation of gradle
111+
// we use this information to determine when the last task that required the cluster executed so that we can
112+
// terminate the cluster right away and free up resources.
113+
configureClaimClustersHook(project.getGradle(), registry);
114+
115+
// Before each task, we determine if a cluster needs to be started for that task.
116+
configureStartClustersHook(project.getGradle(), registry);
117+
118+
// After each task we determine if there are clusters that are no longer needed.
119+
configureStopClustersHook(project.getGradle(), registry);
120+
}
110121

111-
private static void configureStartClustersHook(Gradle gradle, TestClustersRegistry registry) {
112-
gradle.addListener(new TaskActionListener() {
113-
@Override
114-
public void beforeActions(Task task) {
115-
if (task instanceof TestClustersAware == false) {
116-
return;
122+
private static void configureClaimClustersHook(Gradle gradle, TestClustersRegistry registry) {
123+
// Once we know all the tasks that need to execute, we claim all the clusters that belong to those and count the
124+
// claims so we'll know when it's safe to stop them.
125+
gradle.getTaskGraph().whenReady(taskExecutionGraph -> {
126+
taskExecutionGraph.getAllTasks()
127+
.stream()
128+
.filter(task -> task instanceof TestClustersAware)
129+
.map(task -> (TestClustersAware) task)
130+
.flatMap(task -> task.getClusters().stream())
131+
.forEach(registry::claimCluster);
132+
});
133+
}
134+
135+
private static void configureStartClustersHook(Gradle gradle, TestClustersRegistry registry) {
136+
gradle.addListener(new TaskActionListener() {
137+
@Override
138+
public void beforeActions(Task task) {
139+
if (task instanceof TestClustersAware == false) {
140+
return;
141+
}
142+
// we only start the cluster before the actions, so we'll not start it if the task is up-to-date
143+
TestClustersAware awareTask = (TestClustersAware) task;
144+
awareTask.beforeStart();
145+
awareTask.getClusters().forEach(registry::maybeStartCluster);
117146
}
118-
// we only start the cluster before the actions, so we'll not start it if the task is up-to-date
119-
TestClustersAware awareTask = (TestClustersAware) task;
120-
awareTask.beforeStart();
121-
awareTask.getClusters().forEach(registry::maybeStartCluster);
122-
}
123147

124-
@Override
125-
public void afterActions(Task task) {}
126-
});
127-
}
148+
@Override
149+
public void afterActions(Task task) {}
150+
});
151+
}
128152

129-
private static void configureStopClustersHook(Gradle gradle, TestClustersRegistry registry) {
130-
gradle.addListener(new TaskExecutionListener() {
131-
@Override
132-
public void afterExecute(Task task, TaskState state) {
133-
if (task instanceof TestClustersAware == false) {
134-
return;
153+
private static void configureStopClustersHook(Gradle gradle, TestClustersRegistry registry) {
154+
gradle.addListener(new TaskExecutionListener() {
155+
@Override
156+
public void afterExecute(Task task, TaskState state) {
157+
if (task instanceof TestClustersAware == false) {
158+
return;
159+
}
160+
// always unclaim the cluster, even if _this_ task is up-to-date, as others might not have been
161+
// and caused the cluster to start.
162+
((TestClustersAware) task).getClusters().forEach(cluster -> registry.stopCluster(cluster, state.getFailure() != null));
135163
}
136-
// always unclaim the cluster, even if _this_ task is up-to-date, as others might not have been
137-
// and caused the cluster to start.
138-
((TestClustersAware) task).getClusters().forEach(cluster -> registry.stopCluster(cluster, state.getFailure() != null));
139-
}
140164

141-
@Override
142-
public void beforeExecute(Task task) {}
143-
});
165+
@Override
166+
public void beforeExecute(Task task) {}
167+
});
168+
}
144169
}
145-
146170
}

‎buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersRegistry.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,15 @@
22

33
import org.gradle.api.logging.Logger;
44
import org.gradle.api.logging.Logging;
5+
import org.gradle.api.services.BuildService;
6+
import org.gradle.api.services.BuildServiceParameters;
57

68
import java.util.HashMap;
79
import java.util.HashSet;
810
import java.util.Map;
911
import java.util.Set;
1012

11-
public class TestClustersRegistry {
13+
public abstract class TestClustersRegistry implements BuildService<BuildServiceParameters.None> {
1214
private static final Logger logger = Logging.getLogger(TestClustersRegistry.class);
1315
private static final String TESTCLUSTERS_INSPECT_FAILURE = "testclusters.inspect.failure";
1416
private final Boolean allowClusterToSurvive = Boolean.valueOf(System.getProperty(TESTCLUSTERS_INSPECT_FAILURE, "false"));

‎buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersThrottle.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
package org.elasticsearch.gradle.testclusters;
2+
3+
import org.gradle.api.services.BuildService;
4+
import org.gradle.api.services.BuildServiceParameters;
5+
6+
public abstract class TestClustersThrottle implements BuildService<BuildServiceParameters.None> {}

‎buildSrc/src/main/java/org/elasticsearch/gradle/tool/Boilerplate.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,17 @@
1919
package org.elasticsearch.gradle.tool;
2020

2121
import org.gradle.api.Action;
22+
import org.gradle.api.GradleException;
2223
import org.gradle.api.NamedDomainObjectContainer;
2324
import org.gradle.api.PolymorphicDomainObjectContainer;
2425
import org.gradle.api.Project;
2526
import org.gradle.api.Task;
2627
import org.gradle.api.UnknownTaskException;
2728
import org.gradle.api.plugins.JavaPluginConvention;
29+
import org.gradle.api.provider.Provider;
30+
import org.gradle.api.services.BuildService;
31+
import org.gradle.api.services.BuildServiceRegistration;
32+
import org.gradle.api.services.BuildServiceRegistry;
2833
import org.gradle.api.tasks.SourceSetContainer;
2934
import org.gradle.api.tasks.TaskContainer;
3035
import org.gradle.api.tasks.TaskProvider;
@@ -102,4 +107,14 @@ public static TaskProvider<?> findByName(TaskContainer tasks, String name) {
102107

103108
return task;
104109
}
110+
111+
@SuppressWarnings("unchecked")
112+
public static <T extends BuildService<?>> Provider<T> getBuildService(BuildServiceRegistry registry, String name) {
113+
BuildServiceRegistration<?, ?> registration = registry.getRegistrations().findByName(name);
114+
if (registration == null) {
115+
throw new GradleException("Unable to find build service with name '" + name + "'.");
116+
}
117+
118+
return (Provider<T>) registration.getService();
119+
}
105120
}

‎distribution/docker/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -208,6 +208,7 @@ subprojects { Project subProject ->
208208
def tarFile = "${parent.projectDir}/build/elasticsearch${oss ? '-oss' : ''}_test.${VersionProperties.elasticsearch}.docker.tar"
209209

210210
final Task exportDockerImageTask = task(exportTaskName, type: LoggedExec) {
211+
inputs.file("${parent.projectDir}/build/markers/${buildTaskName}.marker")
211212
executable 'docker'
212213
outputs.file(tarFile)
213214
args "save",

‎distribution/docker/src/docker/Dockerfile‎

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,17 @@ RUN chmod 0775 config data logs
3535
COPY config/elasticsearch.yml config/log4j2.properties config/
3636
RUN chmod 0660 config/elasticsearch.yml config/log4j2.properties
3737

38+
# `tini` is a tiny but valid init for containers. This is used to cleanly
39+
# control how ES and any child processes are shut down.
40+
#
41+
# The tini GitHub page gives instructions for verifying the binary using
42+
# gpg, but the keyservers are slow to return the key and this can fail the
43+
# build. Instead, we check the binary against a checksum that we have
44+
# computed.
45+
ADD https://github.com/krallin/tini/releases/download/v0.18.0/tini /tini
46+
COPY config/tini.sha512 /tini.sha512
47+
RUN sha512sum -c /tini.sha512 && chmod +x /tini
48+
3849
################################################################################
3950
# Build stage 1 (the actual elasticsearch image):
4051
# Copy elasticsearch from stage 0
@@ -45,6 +56,8 @@ FROM centos:7
4556

4657
ENV ELASTIC_CONTAINER true
4758

59+
COPY --from=builder /tini /tini
60+
4861
RUN for iter in {1..10}; do yum update --setopt=tsflags=nodocs -y && \
4962
yum install --setopt=tsflags=nodocs -y nc shadow-utils zip unzip && \
5063
yum clean all && exit_code=0 && break || exit_code=\$? && echo "yum error: retry \$iter in 10s" && sleep 10; done; \
@@ -65,14 +78,14 @@ RUN ln -sf /etc/pki/ca-trust/extracted/java/cacerts /usr/share/elasticsearch/jdk
6578

6679
ENV PATH /usr/share/elasticsearch/bin:\$PATH
6780

68-
COPY --chown=1000:0 bin/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
81+
COPY bin/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
6982

70-
# Openshift overrides USER and uses ones with randomly uid>1024 and gid=0
71-
# Allow ENTRYPOINT (and ES) to run even with a different user
72-
RUN chgrp 0 /usr/local/bin/docker-entrypoint.sh && \
73-
chmod g=u /etc/passwd && \
83+
RUN chmod g=u /etc/passwd && \
7484
chmod 0775 /usr/local/bin/docker-entrypoint.sh
7585

86+
# Ensure that there are no files with setuid or setgid, in order to mitigate "stackclash" attacks.
87+
RUN find / -xdev -perm -4000 -exec chmod ug-s {} +
88+
7689
EXPOSE 9200 9300
7790

7891
LABEL org.label-schema.build-date="${build_date}" \
@@ -95,7 +108,9 @@ LABEL org.label-schema.build-date="${build_date}" \
95108
org.opencontainers.image.vendor="Elastic" \
96109
org.opencontainers.image.version="${version}"
97110

98-
ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
111+
USER elasticsearch:root
112+
113+
ENTRYPOINT ["/tini", "--", "/usr/local/bin/docker-entrypoint.sh"]
99114
# Dummy overridable parameter parsed by entrypoint
100115
CMD ["eswrapper"]
101116

0 commit comments

Comments
 (0)