Skip to content

QA: Switch xpack rolling upgrades to three nodes #31112

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jun 6, 2018
Merged

QA: Switch xpack rolling upgrades to three nodes #31112

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9403239
QA: Switch xpack rolling upgrades to three nodes
nik9000 Jun 5, 2018
5f4c90e
Explain copy and paste
nik9000 Jun 5, 2018
55d5784
Merge branch 'master' into xpack_rolling_3
nik9000 Jun 6, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,10 @@
* Basic test that indexed documents survive the rolling restart. See
* {@link RecoveryIT} for much more in depth testing of the mechanism
* by which they survive.
* <p>
* This test is an almost exact copy of <code>IndexingIT</code> in the
* xpack rolling restart tests. We should work on a way to remove this
* duplication but for now we have no real way to share code.
*/
public class IndexingIT extends AbstractRollingTestCase {
public void testIndexing() throws IOException {
Expand Down
144 changes: 48 additions & 96 deletions x-pack/qa/rolling-upgrade-basic/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,62 +8,9 @@ apply plugin: 'elasticsearch.standalone-test'

dependencies {
testCompile project(path: xpackModule('core'), configuration: 'runtime')
testCompile project(path: xpackModule('security'), configuration: 'runtime')
testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') // to be moved in a later commit
}

Closure waitWithAuth = { NodeInfo node, AntBuilder ant ->
File tmpFile = new File(node.cwd, 'wait.success')

// wait up to two minutes
final long stopTime = System.currentTimeMillis() + (2 * 60000L);
Exception lastException = null;
int lastResponseCode = 0

while (System.currentTimeMillis() < stopTime) {

lastException = null;
// we use custom wait logic here as the elastic user is not available immediately and ant.get will fail when a 401 is returned
HttpURLConnection httpURLConnection = null;
try {
// TODO this sucks having to hardcode number of nodes, but node.config.numNodes isn't necessarily accurate for rolling
httpURLConnection = (HttpURLConnection) new URL("http://${node.httpUri()}/_cluster/health?wait_for_nodes=2&wait_for_status=yellow").openConnection();
httpURLConnection.setRequestProperty("Authorization", "Basic " +
Base64.getEncoder().encodeToString("test_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8)));
httpURLConnection.setRequestMethod("GET");
httpURLConnection.setConnectTimeout(1000);
httpURLConnection.setReadTimeout(30000); // read needs to wait for nodes!
httpURLConnection.connect();
lastResponseCode = httpURLConnection.getResponseCode()
if (lastResponseCode == 200) {
tmpFile.withWriter StandardCharsets.UTF_8.name(), {
it.write(httpURLConnection.getInputStream().getText(StandardCharsets.UTF_8.name()))
}
break;
}
} catch (Exception e) {
logger.debug("failed to call cluster health", e)
lastException = e
} finally {
if (httpURLConnection != null) {
httpURLConnection.disconnect();
}
}

// did not start, so wait a bit before trying again
Thread.sleep(500L);
}
if (tmpFile.exists() == false) {
final String message = "final attempt of calling cluster health failed [lastResponseCode=${lastResponseCode}]"
if (lastException != null) {
logger.error(message, lastException)
} else {
logger.error(message + " [no exception]")
}
}
return tmpFile.exists()
}

// This is a top level task which we will add dependencies to below.
// It is a single task that can be used to backcompat tests against all versions.
task bwcTest {
Expand All @@ -82,14 +29,13 @@ for (Version version : bwcVersions.wireCompatible) {

configure(extensions.findByName("${baseName}#oldClusterTestCluster")) {
if (version.before('6.3.0')) {
mavenPlugin 'x-pack', "org.elasticsearch.plugin:x-pack:${version}"
mavenPlugin 'x-pack', "org.elasticsearch.plugin:x-pack:${version}"
}
bwcVersion = version
numBwcNodes = 2
numNodes = 2
minimumMasterNodes = { 2 }
numBwcNodes = 3
numNodes = 3
minimumMasterNodes = { 3 }
clusterName = 'rolling-upgrade-basic'
waitCondition = waitWithAuth
setting 'xpack.security.enabled', 'false'
setting 'xpack.monitoring.enabled', 'false'
setting 'xpack.ml.enabled', 'false'
Expand All @@ -102,51 +48,62 @@ for (Version version : bwcVersions.wireCompatible) {
systemProperty 'tests.rest.suite', 'old_cluster'
}

Task mixedClusterTest = tasks.create(name: "${baseName}#mixedClusterTest", type: RestIntegTestTask)
Closure configureUpgradeCluster = {String name, Task lastRunner, int stopNode, Closure unicastSeed ->
configure(extensions.findByName("${baseName}#${name}")) {
dependsOn lastRunner, "${baseName}#oldClusterTestCluster#node${stopNode}.stop"
clusterName = 'rolling-upgrade-basic'
unicastTransportUri = { seedNode, node, ant -> unicastSeed() }
minimumMasterNodes = { 3 }
/* Override the data directory so the new node always gets the node we
* just stopped's data directory. */
dataDir = { nodeNumber -> oldClusterTest.nodes[stopNode].dataDir }
setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
setting 'xpack.security.enabled', 'false'
setting 'xpack.monitoring.enabled', 'false'
setting 'xpack.ml.enabled', 'false'
setting 'xpack.watcher.enabled', 'false'
setting 'xpack.license.self_generated.type', 'basic'
}
}

Task oneThirdUpgradedTest = tasks.create(name: "${baseName}#oneThirdUpgradedTest", type: RestIntegTestTask)

configure(extensions.findByName("${baseName}#mixedClusterTestCluster")) {
dependsOn oldClusterTestRunner, "${baseName}#oldClusterTestCluster#node1.stop"
clusterName = 'rolling-upgrade-basic'
unicastTransportUri = { seedNode, node, ant -> oldClusterTest.nodes.get(0).transportUri() }
minimumMasterNodes = { 2 }
dataDir = { nodeNumber -> oldClusterTest.nodes[1].dataDir }
waitCondition = waitWithAuth
setting 'xpack.security.enabled', 'false'
setting 'xpack.monitoring.enabled', 'false'
setting 'xpack.ml.enabled', 'false'
setting 'xpack.watcher.enabled', 'false'
setting 'xpack.license.self_generated.type', 'basic'
setting 'node.name', 'mixed-node-0'
configureUpgradeCluster("oneThirdUpgradedTestCluster", oldClusterTestRunner,
0, { oldClusterTest.nodes.get(1).transportUri() })

Task oneThirdUpgradedTestRunner = tasks.getByName("${baseName}#oneThirdUpgradedTestRunner")
oneThirdUpgradedTestRunner.configure {
systemProperty 'tests.rest.suite', 'mixed_cluster'
systemProperty 'tests.first_round', 'true'
finalizedBy "${baseName}#oldClusterTestCluster#node1.stop"
}

Task mixedClusterTestRunner = tasks.getByName("${baseName}#mixedClusterTestRunner")
mixedClusterTestRunner.configure {
Task twoThirdsUpgradedTest = tasks.create(name: "${baseName}#twoThirdsUpgradedTest", type: RestIntegTestTask)

configureUpgradeCluster("twoThirdsUpgradedTestCluster", oneThirdUpgradedTestRunner,
1, { oneThirdUpgradedTest.nodes.get(0).transportUri() })

Task twoThirdsUpgradedTestRunner = tasks.getByName("${baseName}#twoThirdsUpgradedTestRunner")
twoThirdsUpgradedTestRunner.configure {
systemProperty 'tests.rest.suite', 'mixed_cluster'
finalizedBy "${baseName}#oldClusterTestCluster#node0.stop"
systemProperty 'tests.first_round', 'false'
finalizedBy "${baseName}#oldClusterTestCluster#node2.stop"
}

Task upgradedClusterTest = tasks.create(name: "${baseName}#upgradedClusterTest", type: RestIntegTestTask)

configure(extensions.findByName("${baseName}#upgradedClusterTestCluster")) {
dependsOn(mixedClusterTestRunner, "${baseName}#oldClusterTestCluster#node0.stop")
clusterName = 'rolling-upgrade-basic'
unicastTransportUri = { seedNode, node, ant -> mixedClusterTest.nodes.get(0).transportUri() }
minimumMasterNodes = { 2 }
dataDir = { nodeNumber -> oldClusterTest.nodes[0].dataDir }
waitCondition = waitWithAuth
setting 'xpack.security.enabled', 'false'
setting 'xpack.monitoring.enabled', 'false'
setting 'xpack.ml.enabled', 'false'
setting 'xpack.watcher.enabled', 'false'
setting 'xpack.license.self_generated.type', 'basic'
setting 'node.name', 'upgraded-node-0'
}
configureUpgradeCluster("upgradedClusterTestCluster", twoThirdsUpgradedTestRunner,
2, { twoThirdsUpgradedTest.nodes.get(0).transportUri() })

Task upgradedClusterTestRunner = tasks.getByName("${baseName}#upgradedClusterTestRunner")
upgradedClusterTestRunner.configure {
systemProperty 'tests.rest.suite', 'upgraded_cluster'
// only need to kill the mixed cluster tests node here because we explicitly told it to not stop nodes upon completion
finalizedBy "${baseName}#mixedClusterTestCluster#stop"
/*
* Force stopping all the upgraded nodes after the test runner
* so they are alive during the test.
*/
finalizedBy "${baseName}#oneThirdUpgradedTestCluster#stop"
finalizedBy "${baseName}#twoThirdsUpgradedTestCluster#stop"
}

Task versionBwcTest = tasks.create(name: "${baseName}#bwcTest") {
Expand All @@ -170,11 +127,6 @@ task integTest {
}
check.dependsOn(integTest)

dependencies {
testCompile project(path: xpackModule('core'), configuration: 'runtime')
testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
}

compileTestJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked"

// copy x-pack plugin info so it is on the classpath and security manager has the right permissions
Expand Down
136 changes: 70 additions & 66 deletions x-pack/qa/rolling-upgrade/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ Closure waitWithAuth = { NodeInfo node, AntBuilder ant ->
HttpURLConnection httpURLConnection = null;
try {
// TODO this sucks having to hardcode number of nodes, but node.config.numNodes isn't necessarily accurate for rolling
httpURLConnection = (HttpURLConnection) new URL("http://${node.httpUri()}/_cluster/health?wait_for_nodes=2&wait_for_status=yellow").openConnection();
httpURLConnection = (HttpURLConnection) new URL("http://${node.httpUri()}/_cluster/health?wait_for_nodes=3&wait_for_status=yellow").openConnection();
httpURLConnection.setRequestProperty("Authorization", "Basic " +
Base64.getEncoder().encodeToString("test_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8)));
httpURLConnection.setRequestMethod("GET");
Expand Down Expand Up @@ -128,9 +128,9 @@ subprojects {
String usersCli = version.before('6.3.0') ? 'bin/x-pack/users' : 'bin/elasticsearch-users'
setupCommand 'setupTestUser', usersCli, 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'superuser'
bwcVersion = version
numBwcNodes = 2
numNodes = 2
minimumMasterNodes = { 2 }
numBwcNodes = 3
numNodes = 3
minimumMasterNodes = { 3 }
clusterName = 'rolling-upgrade'
waitCondition = waitWithAuth
setting 'xpack.monitoring.exporters._http.type', 'http'
Expand Down Expand Up @@ -167,78 +167,84 @@ subprojects {
systemProperty 'tests.rest.suite', 'old_cluster'
}

Task mixedClusterTest = tasks.create(name: "${baseName}#mixedClusterTest", type: RestIntegTestTask)

configure(extensions.findByName("${baseName}#mixedClusterTestCluster")) {
dependsOn oldClusterTestRunner, "${baseName}#oldClusterTestCluster#node1.stop"
setupCommand 'setupTestUser', 'bin/elasticsearch-users', 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'superuser'
clusterName = 'rolling-upgrade'
unicastTransportUri = { seedNode, node, ant -> oldClusterTest.nodes.get(0).transportUri() }
minimumMasterNodes = { 2 }
dataDir = { nodeNumber -> oldClusterTest.nodes[1].dataDir }
waitCondition = waitWithAuth
setting 'xpack.monitoring.exporters._http.type', 'http'
setting 'xpack.monitoring.exporters._http.enabled', 'false'
setting 'xpack.monitoring.exporters._http.auth.username', 'test_user'
setting 'xpack.monitoring.exporters._http.auth.password', 'x-pack-test-password'
setting 'xpack.license.self_generated.type', 'trial'
setting 'xpack.security.enabled', 'true'
setting 'xpack.security.transport.ssl.enabled', 'true'
setting 'xpack.ssl.keystore.path', 'testnode.jks'
keystoreSetting 'xpack.ssl.keystore.secure_password', 'testnode'
setting 'node.attr.upgraded', 'first'
setting 'xpack.security.authc.token.enabled', 'true'
setting 'xpack.security.audit.enabled', 'true'
setting 'xpack.security.audit.outputs', 'index'
setting 'node.name', 'mixed-node-0'
dependsOn copyTestNodeKeystore
extraConfigFile 'testnode.jks', new File(outputDir + '/testnode.jks')
if (withSystemKey) {
setting 'xpack.watcher.encrypt_sensitive_data', 'true'
keystoreFile 'xpack.watcher.encryption_key', "${mainProject.projectDir}/src/test/resources/system_key"
Closure configureUpgradeCluster = {String name, Task lastRunner, int stopNode, Closure unicastSeed ->
configure(extensions.findByName("${baseName}#${name}")) {
dependsOn lastRunner, "${baseName}#oldClusterTestCluster#node${stopNode}.stop"
setupCommand 'setupTestUser', 'bin/elasticsearch-users', 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'superuser'
clusterName = 'rolling-upgrade'
unicastTransportUri = { seedNode, node, ant -> unicastSeed() }
minimumMasterNodes = { 3 }
/* Override the data directory so the new node always gets the node we
* just stopped's data directory. */
dataDir = { nodeNumber -> oldClusterTest.nodes[stopNode].dataDir }
waitCondition = waitWithAuth
setting 'xpack.monitoring.exporters._http.type', 'http'
setting 'xpack.monitoring.exporters._http.enabled', 'false'
setting 'xpack.monitoring.exporters._http.auth.username', 'test_user'
setting 'xpack.monitoring.exporters._http.auth.password', 'x-pack-test-password'
setting 'xpack.license.self_generated.type', 'trial'
setting 'xpack.security.enabled', 'true'
setting 'xpack.security.transport.ssl.enabled', 'true'
setting 'xpack.ssl.keystore.path', 'testnode.jks'
keystoreSetting 'xpack.ssl.keystore.secure_password', 'testnode'
setting 'node.attr.upgraded', 'true'
setting 'xpack.security.authc.token.enabled', 'true'
setting 'xpack.security.audit.enabled', 'true'
setting 'xpack.security.audit.outputs', 'index'
setting 'node.name', "upgraded-node-${stopNode}"
dependsOn copyTestNodeKeystore
extraConfigFile 'testnode.jks', new File(outputDir + '/testnode.jks')
if (withSystemKey) {
setting 'xpack.watcher.encrypt_sensitive_data', 'true'
keystoreFile 'xpack.watcher.encryption_key', "${mainProject.projectDir}/src/test/resources/system_key"
}
}
}

Task mixedClusterTestRunner = tasks.getByName("${baseName}#mixedClusterTestRunner")
mixedClusterTestRunner.configure {
Task oneThirdUpgradedTest = tasks.create(name: "${baseName}#oneThirdUpgradedTest", type: RestIntegTestTask)

configureUpgradeCluster("oneThirdUpgradedTestCluster", oldClusterTestRunner,
0, { oldClusterTest.nodes.get(1).transportUri() })

Task oneThirdUpgradedTestRunner = tasks.getByName("${baseName}#oneThirdUpgradedTestRunner")
oneThirdUpgradedTestRunner.configure {
systemProperty 'tests.rest.suite', 'mixed_cluster'
finalizedBy "${baseName}#oldClusterTestCluster#node0.stop"
systemProperty 'tests.first_round', 'true'
// We only need to run these tests once so we may as well do it when we're two thirds upgraded
systemProperty 'tests.rest.blacklist', [
'mixed_cluster/10_basic/Start scroll in mixed cluster on upgraded node that we will continue after upgrade',
'mixed_cluster/30_ml_jobs_crud/Create a job in the mixed cluster and write some data',
'mixed_cluster/40_ml_datafeed_crud/Put job and datafeed in mixed cluster',
].join(',')
finalizedBy "${baseName}#oldClusterTestCluster#node1.stop"
}

Task upgradedClusterTest = tasks.create(name: "${baseName}#upgradedClusterTest", type: RestIntegTestTask)
Task twoThirdsUpgradedTest = tasks.create(name: "${baseName}#twoThirdsUpgradedTest", type: RestIntegTestTask)

configure(extensions.findByName("${baseName}#upgradedClusterTestCluster")) {
dependsOn(mixedClusterTestRunner, "${baseName}#oldClusterTestCluster#node0.stop")
setupCommand 'setupTestUser', 'bin/elasticsearch-users', 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'superuser'
clusterName = 'rolling-upgrade'
unicastTransportUri = { seedNode, node, ant -> mixedClusterTest.nodes.get(0).transportUri() }
minimumMasterNodes = { 2 }
dataDir = { nodeNumber -> oldClusterTest.nodes[0].dataDir }
waitCondition = waitWithAuth
setting 'xpack.monitoring.exporters._http.type', 'http'
setting 'xpack.monitoring.exporters._http.enabled', 'false'
setting 'xpack.monitoring.exporters._http.auth.username', 'test_user'
setting 'xpack.monitoring.exporters._http.auth.password', 'x-pack-test-password'
setting 'xpack.license.self_generated.type', 'trial'
setting 'xpack.security.enabled', 'true'
setting 'xpack.security.transport.ssl.enabled', 'true'
setting 'xpack.ssl.keystore.path', 'testnode.jks'
keystoreSetting 'xpack.ssl.keystore.secure_password', 'testnode'
setting 'xpack.security.authc.token.enabled', 'true'
setting 'xpack.security.audit.enabled', 'true'
setting 'xpack.security.audit.outputs', 'index'
setting 'node.name', 'upgraded-node-0'
dependsOn copyTestNodeKeystore
extraConfigFile 'testnode.jks', new File(outputDir + '/testnode.jks')
if (withSystemKey) {
setting 'xpack.watcher.encrypt_sensitive_data', 'true'
keystoreFile 'xpack.watcher.encryption_key', "${mainProject.projectDir}/src/test/resources/system_key"
}
configureUpgradeCluster("twoThirdsUpgradedTestCluster", oneThirdUpgradedTestRunner,
1, { oneThirdUpgradedTest.nodes.get(0).transportUri() })

Task twoThirdsUpgradedTestRunner = tasks.getByName("${baseName}#twoThirdsUpgradedTestRunner")
twoThirdsUpgradedTestRunner.configure {
systemProperty 'tests.rest.suite', 'mixed_cluster'
systemProperty 'tests.first_round', 'false'
finalizedBy "${baseName}#oldClusterTestCluster#node2.stop"
}

Task upgradedClusterTest = tasks.create(name: "${baseName}#upgradedClusterTest", type: RestIntegTestTask)

configureUpgradeCluster("upgradedClusterTestCluster", twoThirdsUpgradedTestRunner,
2, { twoThirdsUpgradedTest.nodes.get(0).transportUri() })

Task upgradedClusterTestRunner = tasks.getByName("${baseName}#upgradedClusterTestRunner")
upgradedClusterTestRunner.configure {
systemProperty 'tests.rest.suite', 'upgraded_cluster'
/*
* Force stopping all the upgraded nodes after the test runner
* so they are alive during the test.
*/
finalizedBy "${baseName}#oneThirdUpgradedTestCluster#stop"
finalizedBy "${baseName}#twoThirdsUpgradedTestCluster#stop"

// migration tests should only run when the original/old cluster nodes where versions < 5.2.0.
// this stinks but we do the check here since our rest tests do not support conditionals
Expand All @@ -251,8 +257,6 @@ subprojects {
systemProperty 'tests.rest.blacklist', '/20_security/Verify default password migration results in upgraded cluster'
}
}
// only need to kill the mixed cluster tests node here because we explicitly told it to not stop nodes upon completion
finalizedBy "${baseName}#mixedClusterTestCluster#stop"
}

Task versionBwcTest = tasks.create(name: "${baseName}#bwcTest") {
Expand Down
Loading