Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,9 @@ public class PluginBuildPlugin extends BuildPlugin {
if (isModule) {
project.integTestCluster.module(project)
project.tasks.run.clusterConfig.module(project)
project.tasks.run.clusterConfig.distribution = 'integ-test-zip'
project.tasks.run.clusterConfig.distribution = System.getProperty(
'run.distribution', 'integ-test-zip'
)
} else {
project.integTestCluster.plugin(project.path)
project.tasks.run.clusterConfig.plugin(project.path)
Expand Down Expand Up @@ -111,7 +113,7 @@ public class PluginBuildPlugin extends BuildPlugin {
private static void createIntegTestTask(Project project) {
RestIntegTestTask integTest = project.tasks.create('integTest', RestIntegTestTask.class)
integTest.mustRunAfter(project.precommit, project.test)
project.integTestCluster.distribution = 'integ-test-zip'
project.integTestCluster.distribution = System.getProperty('tests.distribution', 'integ-test-zip')
project.check.dependsOn(integTest)
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,9 @@ class ClusterFormationTasks {
Configuration currentDistro = project.configurations.create("${prefix}_elasticsearchDistro")
Configuration bwcDistro = project.configurations.create("${prefix}_elasticsearchBwcDistro")
Configuration bwcPlugins = project.configurations.create("${prefix}_elasticsearchBwcPlugins")
if (System.getProperty('tests.distribution', 'oss-zip') == 'integ-test-zip') {
throw new Exception("tests.distribution=integ-test-zip is not supported")
}
configureDistributionDependency(project, config.distribution, currentDistro, VersionProperties.elasticsearch)
if (config.numBwcNodes > 0) {
if (config.bwcVersion == null) {
Expand Down Expand Up @@ -533,7 +536,8 @@ class ClusterFormationTasks {

static Task configureInstallModuleTask(String name, Project project, Task setup, NodeInfo node, Project module) {
if (node.config.distribution != 'integ-test-zip') {
throw new GradleException("Module ${module.path} not allowed be installed distributions other than integ-test-zip because they should already have all modules bundled!")
project.logger.lifecycle("Not installing modules for $name, ${node.config.distribution} already has them")
return setup
}
if (module.plugins.hasPlugin(PluginBuildPlugin) == false) {
throw new GradleException("Task ${name} cannot include module ${module.path} which is not an esplugin")
Expand Down Expand Up @@ -643,6 +647,9 @@ class ClusterFormationTasks {
BuildPlugin.requireJavaHome(start, node.javaVersion)
}
start.doLast(elasticsearchRunner)
start.doFirst {
project.logger.info("Starting node in ${node.clusterName} distribution: ${node.config.distribution}")
}
return start
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,4 @@
- do:
nodes.info: {}

- match: { nodes.$master.modules.0.name: aggs-matrix-stats }
- contains: { nodes.$master.modules: { name: aggs-matrix-stats } }
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,4 @@
- do:
nodes.info: {}

- match: { nodes.$master.modules.0.name: analysis-common }
- contains: { nodes.$master.modules: { name: analysis-common } }
Original file line number Diff line number Diff line change
Expand Up @@ -8,25 +8,25 @@
- do:
nodes.info: {}

- match: { nodes.$master.modules.0.name: ingest-common }
- match: { nodes.$master.ingest.processors.0.type: append }
- match: { nodes.$master.ingest.processors.1.type: convert }
- match: { nodes.$master.ingest.processors.2.type: date }
- match: { nodes.$master.ingest.processors.3.type: date_index_name }
- match: { nodes.$master.ingest.processors.4.type: dot_expander }
- match: { nodes.$master.ingest.processors.5.type: fail }
- match: { nodes.$master.ingest.processors.6.type: foreach }
- match: { nodes.$master.ingest.processors.7.type: grok }
- match: { nodes.$master.ingest.processors.8.type: gsub }
- match: { nodes.$master.ingest.processors.9.type: join }
- match: { nodes.$master.ingest.processors.10.type: json }
- match: { nodes.$master.ingest.processors.11.type: kv }
- match: { nodes.$master.ingest.processors.12.type: lowercase }
- match: { nodes.$master.ingest.processors.13.type: remove }
- match: { nodes.$master.ingest.processors.14.type: rename }
- match: { nodes.$master.ingest.processors.15.type: script }
- match: { nodes.$master.ingest.processors.16.type: set }
- match: { nodes.$master.ingest.processors.17.type: sort }
- match: { nodes.$master.ingest.processors.18.type: split }
- match: { nodes.$master.ingest.processors.19.type: trim }
- match: { nodes.$master.ingest.processors.20.type: uppercase }
- contains: { nodes.$master.modules: { name: ingest-common } }
- contains: { nodes.$master.ingest.processors: { type: append } }
- contains: { nodes.$master.ingest.processors: { type: convert } }
- contains: { nodes.$master.ingest.processors: { type: date } }
- contains: { nodes.$master.ingest.processors: { type: date_index_name } }
- contains: { nodes.$master.ingest.processors: { type: dot_expander } }
- contains: { nodes.$master.ingest.processors: { type: fail } }
- contains: { nodes.$master.ingest.processors: { type: foreach } }
- contains: { nodes.$master.ingest.processors: { type: grok } }
- contains: { nodes.$master.ingest.processors: { type: gsub } }
- contains: { nodes.$master.ingest.processors: { type: join } }
- contains: { nodes.$master.ingest.processors: { type: json } }
- contains: { nodes.$master.ingest.processors: { type: kv } }
- contains: { nodes.$master.ingest.processors: { type: lowercase } }
- contains: { nodes.$master.ingest.processors: { type: remove } }
- contains: { nodes.$master.ingest.processors: { type: rename } }
- contains: { nodes.$master.ingest.processors: { type: script } }
- contains: { nodes.$master.ingest.processors: { type: set } }
- contains: { nodes.$master.ingest.processors: { type: sort } }
- contains: { nodes.$master.ingest.processors: { type: split } }
- contains: { nodes.$master.ingest.processors: { type: trim } }
- contains: { nodes.$master.ingest.processors: { type: uppercase } }
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,4 @@
- do:
nodes.info: {}

- match: { nodes.$master.modules.0.name: lang-expression }
- contains: { nodes.$master.modules: { name: lang-expression } }
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,4 @@
- do:
nodes.info: {}

- match: { nodes.$master.modules.0.name: lang-mustache }
- contains: { nodes.$master.modules: { name: lang-mustache } }
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,4 @@
- do:
nodes.info: {}

- match: { nodes.$master.modules.0.name: lang-painless }
- contains: { nodes.$master.modules: { name: lang-painless } }
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ teardown:
- do:
nodes.info: {}

- match: { nodes.$master.modules.0.name: repository-url }
- contains: { nodes.$master.modules: { name: repository-url } }

---
"Restore with repository-url using http://":
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
- do:
nodes.info: {}

- match: { nodes.$master.modules.0.name: transport-netty4 }
- contains: { nodes.$master.modules: { name: transport-netty4 } }

- do:
cluster.stats: {}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,4 @@
- do:
nodes.info: {}

- match: { nodes.$master.plugins.0.name: discovery-azure-classic }
- contains: { nodes.$master.plugins: { name: discovery-azure-classic } }
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,4 @@
- do:
nodes.info: {}

- match: { nodes.$master.plugins.0.name: discovery-ec2 }
- contains: { nodes.$master.plugins: { name: discovery-ec2 } }
2 changes: 1 addition & 1 deletion plugins/discovery-file/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ task setupSeedNodeAndUnicastHostsFile(type: DefaultTask) {
// setup the initial cluster with one node that will serve as the seed node
// for unicast discovery
ClusterConfiguration config = new ClusterConfiguration(project)
config.distribution = 'integ-test-zip'
config.distribution = System.getProperty('tests.distribution', 'integ-test-zip')
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure we need this line at all given your change to PluginBuildPlugin.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This would take precedence and override that. So it would be set back to integ-test-zip regardless of the property.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am concerned this is fragile. I don't want to see this logic copied around to other projects. Can you please create an issue for making creating additional cluster fixtures easier, so that the default can be set by PluginBuildPlugin for this project and reused the we create this cluster?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

++ on this being fragile. Fixing it in a followup is fine with me though.

config.clusterName = 'discovery-file-test-cluster'
List<NodeInfo> nodes = ClusterFormationTasks.setup(project, 'initialCluster', setupSeedNodeAndUnicastHostsFile, config)
File srcUnicastHostsFile = file('build/cluster/unicast_hosts.txt')
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,4 @@
- do:
nodes.info: {}

- match: { nodes.$master.plugins.0.name: discovery-gce }
- contains: { nodes.$master.plugins: { name: discovery-gce } }
4 changes: 2 additions & 2 deletions plugins/examples/painless-whitelist/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@ dependencies {
compileOnly project(':modules:lang-painless')
}

integTestCluster {
distribution = 'zip'
if (System.getProperty('tests.distribution') == null) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This seems a bit complicated to add to every project. I'm not 100% sure we need to support -Dtests.distribution=integ-test-zip. Maybe integ-test-zip is a thing we should only use when you don't override the property.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It'd be fairly simple to fail the build if someone sets distribution to integ-test-zip. If we did that then this project could do distribution = System.getProperty('tests.distribution', 'zip).

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I might be cleaner to move the PluginBuildPlugin change to a gradle.projectsEvaluated and have it handle the logic all in one place.

Copy link
Contributor Author

@alpar-t alpar-t May 25, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

setting distro integ-test-zip would offer a way to run some integration tests in a build that would be much quicker as it would skip a lot of the heavier integration tests (probably most of QA). I'm not sure how useful that would be during development, so can't decide if it's worth doing.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think that is generally useful. Plugins and modules are the projects which use integ-test-zip, and you can already cd into the plugins or modules dir and run tests from there.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah. I don't think we should support setting -Dtests.distribution=integ-test-zip. It should be what you get when you run without setting it.

integTestCluster.distribution = 'oss-zip'
}

test.enabled = false
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,4 @@
- do:
nodes.info: {}

- match: { nodes.$master.plugins.0.name: painless-whitelist }
- contains: { nodes.$master.plugins: { name: painless-whitelist } }
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,4 @@
- do:
nodes.info: {}

- match: { nodes.$master.plugins.0.name: example-rescore }
- contains: { nodes.$master.plugins: { name: example-rescore } }
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,4 @@
- do:
nodes.info: {}

- match: { nodes.$master.plugins.0.name: script-expert-scoring }
- contains: { nodes.$master.plugins: { name: script-expert-scoring } }
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,6 @@
- do:
nodes.info: {}

- match: { nodes.$master.plugins.0.name: ingest-attachment }
- match: { nodes.$master.ingest.processors.0.type: attachment }
- contains: { 'nodes.$master.plugins': { name: ingest-attachment } }
- contains: { 'nodes.$master.ingest.processors': { type: attachment } }

Original file line number Diff line number Diff line change
Expand Up @@ -7,5 +7,5 @@
- do:
nodes.info: {}

- match: { nodes.$master.plugins.0.name: ingest-geoip }
- match: { nodes.$master.ingest.processors.0.type: geoip }
- contains: { nodes.$master.plugins: { name: ingest-geoip } }
- contains: { nodes.$master.ingest.processors: { type: geoip } }
Original file line number Diff line number Diff line change
Expand Up @@ -7,5 +7,5 @@
- do:
nodes.info: {}

- match: { nodes.$master.plugins.0.name: ingest-user-agent }
- match: { nodes.$master.ingest.processors.0.type: user_agent }
- contains: { nodes.$master.plugins: { name: ingest-user-agent } }
- contains: { nodes.$master.ingest.processors: { type: user_agent } }
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,4 @@
- do:
nodes.info: {}

- match: { nodes.$master.plugins.0.name: repository-azure }
- contains: { nodes.$master.plugins: { name: repository-azure } }
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,4 @@
- do:
nodes.info: {}

- match: { nodes.$master.plugins.0.name: repository-gcs }
- contains: { nodes.$master.plugins: { name: repository-gcs } }
1 change: 0 additions & 1 deletion plugins/repository-hdfs/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,6 @@ for (String fixtureName : ['hdfsFixture', 'haHdfsFixture', 'secureHdfsFixture',
project.afterEvaluate {
for (String integTestTaskName : ['integTestHa', 'integTestSecure', 'integTestSecureHa']) {
ClusterConfiguration cluster = project.extensions.getByName("${integTestTaskName}Cluster") as ClusterConfiguration
cluster.distribution = 'integ-test-zip'
cluster.dependsOn(project.bundlePlugin)

Task restIntegTestTask = project.tasks.getByName(integTestTaskName)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
- do:
nodes.info: {}

- match: { nodes.$master.plugins.0.name: repository-hdfs }
- contains: { nodes.$master.plugins: { name: repository-hdfs } }
---
#
# Check that we can't use file:// repositories or anything like that
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
- do:
nodes.info: {}

- match: { nodes.$master.plugins.0.name: repository-hdfs }
- contains: { nodes.$master.plugins: { name: repository-hdfs } }
---
#
# Check that we can't use file:// repositories or anything like that
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,4 @@
- do:
nodes.info: {}

- match: { nodes.$master.plugins.0.name: repository-s3 }
- contains: { nodes.$master.plugins: { name: repository-s3 } }
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,4 @@
- do:
nodes.info: {}

- match: { nodes.$master.plugins.0.name: store-smb }
- contains: { nodes.$master.plugins: { name: store-smb } }
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.test.rest.yaml.section;

import org.apache.logging.log4j.Logger;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.xcontent.XContentLocation;
import org.elasticsearch.common.xcontent.XContentParser;

import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.not;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

public class ContainsAssertion extends Assertion {
public static ContainsAssertion parse(XContentParser parser) throws IOException {
XContentLocation location = parser.getTokenLocation();
Tuple<String,Object> stringObjectTuple = ParserUtils.parseTuple(parser);
return new ContainsAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2());
}

private static final Logger logger = Loggers.getLogger(ContainsAssertion.class);

public ContainsAssertion(XContentLocation location, String field, Object expectedValue) {
super(location, field, expectedValue);
}

@Override
protected void doAssert(Object actualValue, Object expectedValue) {
// add support for matching objects ({a:b}) against list of objects ([ {a:b, c:d} ])
if(expectedValue instanceof Map && actualValue instanceof List) {
logger.trace("assert that [{}] contains [{}]", actualValue, expectedValue);
Map<String, Object> expectedMap = (Map<String, Object>) expectedValue;
List<Object> actualList = (List<Object>) actualValue;
List<Map<String, Object>> actualValues = actualList.stream()
.filter(each -> each instanceof Map)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we should fail the assertion if the thing we're testing isn't a map.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As implemented now, we can check that a list has an object that has all the keys at the specific values. In theory we could have a list having a mix of lists and objects. The filter only rules out anything that's not a map at this point when looking for actualValues. The subsequent check on actual values not empty would fail the assertion in case of a list of lists for example.
The only case we don't fail if we have a list that has both a list and an object that has the keys we are interested in, but I don't think we should in this case, because we did find that the list contains what we are looking for.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see! Got it.

.map((each -> (Map<String, Object>) each))
.filter(each -> each.keySet().containsAll(expectedMap.keySet()))
.collect(Collectors.toList());
assertThat(
getField() + " expected to be a list with at least one object that has keys: " +
expectedMap.keySet() + " but it was " + actualList,
actualValues,
is(not(empty()))
);
assertTrue(
getField() + " expected to be a list with at least on object that matches " + expectedMap +
" but was " + actualValues,
actualValues.stream()
.anyMatch(each -> each.entrySet().containsAll(expectedMap.entrySet()))
);
} else {
fail("'contains' only supports checking an object against a list of objects");
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ public interface ExecutableSection {
new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("gte"), GreaterThanEqualToAssertion::parse),
new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("lt"), LessThanAssertion::parse),
new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("lte"), LessThanOrEqualToAssertion::parse),
new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("contains"), ContainsAssertion::parse),
new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("length"), LengthAssertion::parse)));

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,22 @@ public void testParseMatchArray() throws Exception {
assertThat(strings.get(1).toString(), equalTo("test_percolator_2"));
}

@SuppressWarnings("unchecked")
public void testParseContains() throws Exception {
parser = createParser(YamlXContent.yamlXContent,
"{testKey: { someKey: someValue } }"
);

ContainsAssertion containsAssertion = ContainsAssertion.parse(parser);
assertThat(containsAssertion, notNullValue());
assertThat(containsAssertion.getField(), equalTo("testKey"));
assertThat(containsAssertion.getExpectedValue(), instanceOf(Map.class));
assertThat(
((Map<String, String>) containsAssertion.getExpectedValue()).get("someKey"),
equalTo("someValue")
);
}

@SuppressWarnings("unchecked")
public void testParseMatchSourceValues() throws Exception {
parser = createParser(YamlXContent.yamlXContent,
Expand Down
4 changes: 4 additions & 0 deletions x-pack/plugin/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -192,3 +192,7 @@ integTestCluster {
return tmpFile.exists()
}
}
if (integTestCluster.distribution.startsWith("oss-") == false) {
integTest.enabled = false
}

Loading