From 49f6ada6070016246bdb610deeedf9128e0dea21 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 16 May 2019 09:56:22 -0400 Subject: [PATCH 1/3] Remove the migrate tool This commit removes the deprecated migrate tool which was used to migrate users from the file realm to native realm when the native realm was first created. --- docs/reference/commands/index.asciidoc | 2 - docs/reference/commands/migrate-tool.asciidoc | 112 ----- .../migration/migrate_8_0/security.asciidoc | 8 + .../src/main/bin/elasticsearch-migrate | 10 - .../src/main/bin/elasticsearch-migrate.bat | 19 - .../esnative/ESNativeRealmMigrateTool.java | 399 ------------------ .../esnative/ESNativeMigrateToolTests.java | 175 -------- .../ESNativeRealmMigrateToolTests.java | 149 ------- .../xpack/security/MigrateToolIT.java | 130 ------ .../xpack/security/MigrateToolTestCase.java | 166 -------- 10 files changed, 8 insertions(+), 1162 deletions(-) delete mode 100644 docs/reference/commands/migrate-tool.asciidoc delete mode 100755 x-pack/plugin/security/src/main/bin/elasticsearch-migrate delete mode 100644 x-pack/plugin/security/src/main/bin/elasticsearch-migrate.bat delete mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeRealmMigrateTool.java delete mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java delete mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeRealmMigrateToolTests.java delete mode 100644 x-pack/qa/security-migrate-tests/src/test/java/org/elasticsearch/xpack/security/MigrateToolIT.java delete mode 100644 x-pack/qa/security-migrate-tests/src/test/java/org/elasticsearch/xpack/security/MigrateToolTestCase.java diff --git a/docs/reference/commands/index.asciidoc b/docs/reference/commands/index.asciidoc index a13ea58c27d3e..e778366aa58b9 100644 --- a/docs/reference/commands/index.asciidoc +++ b/docs/reference/commands/index.asciidoc @@ -9,7 +9,6 @@ tasks from the command line: * <> * <> -* <> * <> * <> * <> @@ -21,7 +20,6 @@ tasks from the command line: include::certgen.asciidoc[] include::certutil.asciidoc[] -include::migrate-tool.asciidoc[] include::node-tool.asciidoc[] include::saml-metadata.asciidoc[] include::setup-passwords.asciidoc[] diff --git a/docs/reference/commands/migrate-tool.asciidoc b/docs/reference/commands/migrate-tool.asciidoc deleted file mode 100644 index 2c2f4abf4333b..0000000000000 --- a/docs/reference/commands/migrate-tool.asciidoc +++ /dev/null @@ -1,112 +0,0 @@ -[role="xpack"] -[testenv="gold+"] -[[migrate-tool]] -== elasticsearch-migrate - -deprecated:[7.2.0, "This tool is deprecated. Use the native realm directly."] - -The `elasticsearch-migrate` command migrates existing file-based users and roles -to the native realm. From 5.0 onward, you should use the `native` realm to -manage roles and local users. - - -[float] -=== Synopsis - -[source,shell] --------------------------------------------------- -bin/elasticsearch-migrate -(native (-U, --url ) -[-h, --help] [-E ] -[-n, --users ] [-r, --roles ] -[-u, --username ] [-p, --password ] -[-s, --silent] [-v, --verbose]) --------------------------------------------------- - -[float] -=== Description - -NOTE: When migrating from Shield 2.x, the `elasticsearch-migrate` tool should be -run prior to upgrading to ensure all roles can be migrated as some may be in a -deprecated format that {xpack} cannot read. The `migrate` tool is available in -Shield 2.4.0 and higher. - -The `elasticsearch-migrate` tool loads the existing file-based users and roles -and calls the user and roles APIs to add them to the native realm. You can -migrate all users and roles, or specify the ones you want to migrate. Users and -roles that already exist in the `native` realm are not replaced or -overridden. If the names you specify with the `--users` and `--roles` options -don't exist in the `file` realm, they are skipped. - -[float] -[[migrate-tool-options]] -=== Parameters -The `native` subcommand supports the following options: - -`-E `:: -Configures a setting. - -`-h, --help`:: -Returns all of the command parameters. - -`-n`, `--users`:: -Comma-separated list of the users that you want to migrate. If this parameter is -not specified, all users are migrated. - -`-p`, `--password`:: -Password to use for authentication with {es}. -//TBD: What is the default if this isn't specified? - -`-r`, `--roles`:: -Comma-separated list of the roles that you want to migrate. If this parameter is -not specified, all roles are migrated. - -`-s, --silent`:: Shows minimal output. - -`-U`, `--url`:: -Endpoint URL of the {es} cluster to which you want to migrate the -file-based users and roles. This parameter is required. - -`-u`, `--username`:: -Username to use for authentication with {es}. -//TBD: What is the default if this isn't specified? - -`-v, --verbose`:: Shows verbose output. - -[float] -=== Examples - -Run the `elasticsearch-migrate` tool when {xpack} is installed. For example: - -[source, sh] ----------------------------------------------------------------------- -$ bin/elasticsearch-migrate native -U http://localhost:9200 -u elastic --p x-pack-test-password -n lee,foo -r role1,role2,role3,role4,foo -starting migration of users and roles... -importing users from [/home/es/config/shield/users]... -found existing users: [test_user, joe3, joe2] -migrating user [lee] -{"user":{"created":true}} -no user [foo] found, skipping -importing roles from [/home/es/config/shield/roles.yml]... -found existing roles: [marvel_user, role_query_fields, admin_role, role3, admin, -remote_marvel_agent, power_user, role_new_format_name_array, role_run_as, -logstash, role_fields, role_run_as1, role_new_format, kibana4_server, user, -transport_client, role1.ab, role_query] -migrating role [role1] -{"role":{"created":true}} -migrating role [role2] -{"role":{"created":true}} -role [role3] already exists, skipping -no role [foo] found, skipping -users and roles imported. ----------------------------------------------------------------------- - -Additionally, the `-E` flag can be used to specify additional settings. For example -to specify a different configuration directory, the command would look like: - -[source, sh] ----------------------------------------------------------------------- -$ bin/elasticsearch-migrate native -U http://localhost:9200 -u elastic --p x-pack-test-password -E path.conf=/etc/elasticsearch ----------------------------------------------------------------------- diff --git a/docs/reference/migration/migrate_8_0/security.asciidoc b/docs/reference/migration/migrate_8_0/security.asciidoc index fcc0a5b22168a..a7cacef8ff017 100644 --- a/docs/reference/migration/migrate_8_0/security.asciidoc +++ b/docs/reference/migration/migrate_8_0/security.asciidoc @@ -25,3 +25,11 @@ The `xpack.security.authz.store.roles.index.cache.max_size` and been removed. These settings have been redundant and deprecated since the 5.2 release of {es}. +[float] +[[migrate-tool-removed]] +==== The `elasticsearch-migrate` tool has been removed + +The `elasticsearch-migrate` tool provided a way to convert file +realm users and roles into the native realm. It has been deprecated +since 7.2.0. Users and roles should now be created in the native +realm directly. diff --git a/x-pack/plugin/security/src/main/bin/elasticsearch-migrate b/x-pack/plugin/security/src/main/bin/elasticsearch-migrate deleted file mode 100755 index 183722d9c9364..0000000000000 --- a/x-pack/plugin/security/src/main/bin/elasticsearch-migrate +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License; -# you may not use this file except in compliance with the Elastic License. - -ES_MAIN_CLASS=org.elasticsearch.xpack.security.authc.esnative.ESNativeRealmMigrateTool \ - ES_ADDITIONAL_SOURCES="x-pack-env;x-pack-security-env" \ - "`dirname "$0"`"/elasticsearch-cli \ - "$@" diff --git a/x-pack/plugin/security/src/main/bin/elasticsearch-migrate.bat b/x-pack/plugin/security/src/main/bin/elasticsearch-migrate.bat deleted file mode 100644 index a50bc1a384ed0..0000000000000 --- a/x-pack/plugin/security/src/main/bin/elasticsearch-migrate.bat +++ /dev/null @@ -1,19 +0,0 @@ -@echo off - -rem Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -rem or more contributor license agreements. Licensed under the Elastic License; -rem you may not use this file except in compliance with the Elastic License. - -setlocal enabledelayedexpansion -setlocal enableextensions - -set ES_MAIN_CLASS=org.elasticsearch.xpack.security.authc.esnative.ESNativeRealmMigrateTool -set ES_ADDITIONAL_SOURCES=x-pack-env;x-pack-security-env -call "%~dp0elasticsearch-cli.bat" ^ - %%* ^ - || goto exit - -endlocal -endlocal -:exit -exit /b %ERRORLEVEL% diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeRealmMigrateTool.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeRealmMigrateTool.java deleted file mode 100644 index 0fbe54d7c1066..0000000000000 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeRealmMigrateTool.java +++ /dev/null @@ -1,399 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.security.authc.esnative; - -import joptsimple.OptionParser; -import joptsimple.OptionSet; -import joptsimple.OptionSpec; -import org.apache.logging.log4j.Level; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.core.Appender; -import org.apache.logging.log4j.core.LogEvent; -import org.apache.logging.log4j.core.LoggerContext; -import org.apache.logging.log4j.core.appender.AbstractAppender; -import org.apache.logging.log4j.core.config.Configuration; -import org.apache.logging.log4j.core.config.LoggerConfig; -import org.apache.logging.log4j.core.layout.PatternLayout; -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.cli.EnvironmentAwareCommand; -import org.elasticsearch.cli.LoggingAwareMultiCommand; -import org.elasticsearch.cli.Terminal; -import org.elasticsearch.cli.Terminal.Verbosity; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.SuppressForbidden; -import org.elasticsearch.common.logging.Loggers; -import org.elasticsearch.common.settings.SecureString; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.env.Environment; -import org.elasticsearch.xpack.core.common.socket.SocketAccess; -import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; -import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; -import org.elasticsearch.xpack.core.ssl.SSLConfiguration; -import org.elasticsearch.xpack.security.authz.store.FileRolesStore; -import org.elasticsearch.xpack.core.ssl.SSLService; -import org.elasticsearch.xpack.security.authc.file.FileUserPasswdStore; -import org.elasticsearch.xpack.security.authc.file.FileUserRolesStore; - -import javax.net.ssl.HttpsURLConnection; - -import java.io.BufferedReader; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.net.HttpURLConnection; -import java.net.URI; -import java.net.URL; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.security.AccessController; -import java.security.PrivilegedAction; -import java.util.Arrays; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; - -/** - * This is the command-line tool used for migrating users and roles from the file-based realm into the new native realm using the API for - * import. It reads from the files and tries its best to add the users, showing an error if it was incapable of importing them. Any existing - * users or roles are skipped. - */ -public class ESNativeRealmMigrateTool extends LoggingAwareMultiCommand { - - public static void main(String[] args) throws Exception { - exit(new ESNativeRealmMigrateTool().main(args, Terminal.DEFAULT)); - } - - public ESNativeRealmMigrateTool() { - super("Imports file-based users and roles to the native security realm"); - subcommands.put("native", newMigrateUserOrRoles()); - } - - protected MigrateUserOrRoles newMigrateUserOrRoles() { - return new MigrateUserOrRoles(); - } - - /** - * Command to migrate users and roles to the native realm - */ - public static class MigrateUserOrRoles extends EnvironmentAwareCommand { - - private final OptionSpec username; - private final OptionSpec password; - private final OptionSpec url; - private final OptionSpec usersToMigrateCsv; - private final OptionSpec rolesToMigrateCsv; - - public MigrateUserOrRoles() { - super("Migrates users or roles from file to native realm"); - this.username = parser.acceptsAll(Arrays.asList("u", "username"), - "User used to authenticate with Elasticsearch") - .withRequiredArg().required(); - this.password = parser.acceptsAll(Arrays.asList("p", "password"), - "Password used to authenticate with Elasticsearch") - .withRequiredArg().required(); - this.url = parser.acceptsAll(Arrays.asList("U", "url"), - "URL of Elasticsearch host") - .withRequiredArg(); - this.usersToMigrateCsv = parser.acceptsAll(Arrays.asList("n", "users"), - "Users to migrate from file to native realm") - .withRequiredArg(); - this.rolesToMigrateCsv = parser.acceptsAll(Arrays.asList("r", "roles"), - "Roles to migrate from file to native realm") - .withRequiredArg(); - } - - // Visible for testing - public OptionParser getParser() { - return this.parser; - } - - @Override - protected void printAdditionalHelp(Terminal terminal) { - terminal.println("This tool migrates file based users[1] and roles[2] to the native realm in"); - terminal.println("elasticsearch, saving the administrator from needing to manually transition"); - terminal.println("them from the file."); - } - - // Visible for testing - @Override - public void execute(Terminal terminal, OptionSet options, Environment env) throws Exception { - terminal.println("Warning: The migrate tool is deprecated. Use the native realm directly instead of file realms."); - terminal.println("starting migration of users and roles..."); - importUsers(terminal, env, options); - importRoles(terminal, env, options); - terminal.println("users and roles imported."); - } - - @SuppressForbidden(reason = "We call connect in doPrivileged and provide SocketPermission") - private String postURL(Settings settings, Environment env, String method, String urlString, - OptionSet options, @Nullable String bodyString) throws Exception { - URI uri = new URI(urlString); - URL url = uri.toURL(); - HttpURLConnection conn; - // If using SSL, need a custom service because it's likely a self-signed certificate - if ("https".equalsIgnoreCase(uri.getScheme())) { - final SSLService sslService = new SSLService(settings, env); - final SSLConfiguration sslConfiguration = sslService.getSSLConfiguration("xpack.security.http.ssl"); - final HttpsURLConnection httpsConn = (HttpsURLConnection) url.openConnection(); - AccessController.doPrivileged((PrivilegedAction) () -> { - // Requires permission java.lang.RuntimePermission "setFactory"; - httpsConn.setSSLSocketFactory(sslService.sslSocketFactory(sslConfiguration)); - return null; - }); - conn = httpsConn; - } else { - conn = (HttpURLConnection) url.openConnection(); - } - conn.setRequestMethod(method); - conn.setReadTimeout(30 * 1000); // 30 second timeout - // Add basic-auth header - conn.setRequestProperty("Authorization", - UsernamePasswordToken.basicAuthHeaderValue(username.value(options), - new SecureString(password.value(options).toCharArray()))); - conn.setRequestProperty("Content-Type", XContentType.JSON.mediaType()); - conn.setDoOutput(true); // we'll be sending a body - SocketAccess.doPrivileged(conn::connect); - if (bodyString != null) { - try (OutputStream out = conn.getOutputStream()) { - out.write(bodyString.getBytes(StandardCharsets.UTF_8)); - } catch (Exception e) { - try { - conn.disconnect(); - } catch (Exception e2) { - // Ignore exceptions if we weren't able to close the connection after an error - } - throw e; - } - } - try (BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getInputStream(), StandardCharsets.UTF_8))) { - StringBuilder sb = new StringBuilder(); - String line = null; - while ((line = reader.readLine()) != null) { - sb.append(line); - } - return sb.toString(); - } catch (IOException e) { - try (BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getErrorStream(), StandardCharsets.UTF_8))) { - StringBuilder sb = new StringBuilder(); - String line = null; - while ((line = reader.readLine()) != null) { - sb.append(line); - } - throw new IOException(sb.toString(), e); - } - } finally { - conn.disconnect(); - } - } - - Set getUsersThatExist(Terminal terminal, Settings settings, Environment env, OptionSet options) throws Exception { - Set existingUsers = new HashSet<>(); - String allUsersJson = postURL(settings, env, "GET", this.url.value(options) + "/_security/user/", options, null); - // EMPTY is safe here because we never use namedObject - try (XContentParser parser = JsonXContent.jsonXContent - .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, allUsersJson)) { - XContentParser.Token token = parser.nextToken(); - String userName; - if (token == XContentParser.Token.START_OBJECT) { - while ((token = parser.nextToken()) == XContentParser.Token.FIELD_NAME) { - userName = parser.currentName(); - existingUsers.add(userName); - parser.nextToken(); - parser.skipChildren(); - } - } else { - throw new ElasticsearchException("failed to retrieve users, expecting an object but got: " + token); - } - } - terminal.println("found existing users: " + existingUsers); - return existingUsers; - } - - static String createUserJson(String[] roles, char[] password) throws IOException { - XContentBuilder builder = jsonBuilder(); - builder.startObject(); - { - builder.field("password_hash", new String(password)); - builder.startArray("roles"); - for (String role : roles) { - builder.value(role); - } - builder.endArray(); - } - builder.endObject(); - return Strings.toString(builder); - } - - void importUsers(Terminal terminal, Environment env, OptionSet options) throws FileNotFoundException { - String usersCsv = usersToMigrateCsv.value(options); - String[] usersToMigrate = (usersCsv != null) ? usersCsv.split(",") : Strings.EMPTY_ARRAY; - Path usersFile = FileUserPasswdStore.resolveFile(env); - Path usersRolesFile = FileUserRolesStore.resolveFile(env); - if (Files.exists(usersFile) == false) { - throw new FileNotFoundException("users file [" + usersFile + "] does not exist"); - } else if (Files.exists(usersRolesFile) == false) { - throw new FileNotFoundException("users_roles file [" + usersRolesFile + "] does not exist"); - } - - terminal.println("importing users from [" + usersFile + "]..."); - final Logger logger = getTerminalLogger(terminal); - Map userToHashedPW = FileUserPasswdStore.parseFile(usersFile, logger, env.settings()); - Map userToRoles = FileUserRolesStore.parseFile(usersRolesFile, logger); - Set existingUsers; - try { - existingUsers = getUsersThatExist(terminal, env.settings(), env, options); - } catch (Exception e) { - throw new ElasticsearchException("failed to get users that already exist, skipping user import", e); - } - if (usersToMigrate.length == 0) { - usersToMigrate = userToHashedPW.keySet().toArray(new String[userToHashedPW.size()]); - } - for (String user : usersToMigrate) { - if (userToHashedPW.containsKey(user) == false) { - terminal.println("user [" + user + "] was not found in files, skipping"); - continue; - } else if (existingUsers.contains(user)) { - terminal.println("user [" + user + "] already exists, skipping"); - continue; - } - terminal.println("migrating user [" + user + "]"); - String reqBody = "n/a"; - try { - reqBody = createUserJson(userToRoles.get(user), userToHashedPW.get(user)); - String resp = postURL(env.settings(), env, "POST", - this.url.value(options) + "/_security/user/" + user, options, reqBody); - terminal.println(resp); - } catch (Exception e) { - throw new ElasticsearchException("failed to migrate user [" + user + "] with body: " + reqBody, e); - } - } - } - - Set getRolesThatExist(Terminal terminal, Settings settings, Environment env, OptionSet options) throws Exception { - Set existingRoles = new HashSet<>(); - String allRolesJson = postURL(settings, env, "GET", this.url.value(options) + "/_security/role/", options, null); - // EMPTY is safe here because we never use namedObject - try (XContentParser parser = JsonXContent.jsonXContent - .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, allRolesJson)) { - XContentParser.Token token = parser.nextToken(); - String roleName; - if (token == XContentParser.Token.START_OBJECT) { - while ((token = parser.nextToken()) == XContentParser.Token.FIELD_NAME) { - roleName = parser.currentName(); - existingRoles.add(roleName); - parser.nextToken(); - parser.skipChildren(); - } - } else { - throw new ElasticsearchException("failed to retrieve roles, expecting an object but got: " + token); - } - } - terminal.println("found existing roles: " + existingRoles); - return existingRoles; - } - - static String createRoleJson(RoleDescriptor rd) throws IOException { - XContentBuilder builder = jsonBuilder(); - rd.toXContent(builder, ToXContent.EMPTY_PARAMS, true); - return Strings.toString(builder); - } - - void importRoles(Terminal terminal, Environment env, OptionSet options) throws FileNotFoundException { - String rolesCsv = rolesToMigrateCsv.value(options); - String[] rolesToMigrate = (rolesCsv != null) ? rolesCsv.split(",") : Strings.EMPTY_ARRAY; - Path rolesFile = FileRolesStore.resolveFile(env).toAbsolutePath(); - if (Files.exists(rolesFile) == false) { - throw new FileNotFoundException("roles.yml file [" + rolesFile + "] does not exist"); - } - terminal.println("importing roles from [" + rolesFile + "]..."); - Logger logger = getTerminalLogger(terminal); - Map roles = FileRolesStore.parseRoleDescriptors(rolesFile, logger, true, Settings.EMPTY); - Set existingRoles; - try { - existingRoles = getRolesThatExist(terminal, env.settings(), env, options); - } catch (Exception e) { - throw new ElasticsearchException("failed to get roles that already exist, skipping role import", e); - } - if (rolesToMigrate.length == 0) { - rolesToMigrate = roles.keySet().toArray(new String[roles.size()]); - } - for (String roleName : rolesToMigrate) { - if (roles.containsKey(roleName) == false) { - terminal.println("no role [" + roleName + "] found, skipping"); - continue; - } else if (existingRoles.contains(roleName)) { - terminal.println("role [" + roleName + "] already exists, skipping"); - continue; - } - terminal.println("migrating role [" + roleName + "]"); - String reqBody = "n/a"; - try { - reqBody = createRoleJson(roles.get(roleName)); - String resp = postURL(env.settings(), env, "POST", - this.url.value(options) + "/_security/role/" + roleName, options, reqBody); - terminal.println(resp); - } catch (Exception e) { - throw new ElasticsearchException("failed to migrate role [" + roleName + "] with body: " + reqBody, e); - } - } - } - } - - /** - * Creates a new Logger that is detached from the ROOT logger and only has an appender that will output log messages to the terminal - */ - static Logger getTerminalLogger(final Terminal terminal) { - final Logger logger = LogManager.getLogger(ESNativeRealmMigrateTool.class); - Loggers.setLevel(logger, Level.ALL); - - final LoggerContext ctx = (LoggerContext) LogManager.getContext(false); - final Configuration config = ctx.getConfiguration(); - - // create appender - final Appender appender = new AbstractAppender(ESNativeRealmMigrateTool.class.getName(), null, - PatternLayout.newBuilder() - // Specify the configuration so log4j doesn't re-initialize - .withConfiguration(config) - .withPattern("%m") - .build()) { - @Override - public void append(LogEvent event) { - switch (event.getLevel().getStandardLevel()) { - case FATAL: - case ERROR: - terminal.println(Verbosity.NORMAL, event.getMessage().getFormattedMessage()); - break; - case OFF: - break; - default: - terminal.println(Verbosity.VERBOSE, event.getMessage().getFormattedMessage()); - break; - } - } - }; - appender.start(); - - // get the config, detach from parent, remove appenders, add custom appender - final LoggerConfig loggerConfig = config.getLoggerConfig(ESNativeRealmMigrateTool.class.getName()); - loggerConfig.setParent(null); - loggerConfig.getAppenders().forEach((s, a) -> Loggers.removeAppender(logger, a)); - Loggers.addAppender(logger, appender); - return logger; - } -} diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java deleted file mode 100644 index a73fc93f32e45..0000000000000 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.security.authc.esnative; - -import joptsimple.OptionException; -import joptsimple.OptionParser; -import joptsimple.OptionSet; -import org.elasticsearch.cli.MockTerminal; -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.env.Environment; -import org.elasticsearch.test.NativeRealmIntegTestCase; -import org.elasticsearch.common.CharArrays; -import org.elasticsearch.xpack.core.security.client.SecurityClient; -import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames; -import org.junit.BeforeClass; - -import java.nio.charset.StandardCharsets; -import java.nio.file.Path; -import java.util.Collections; -import java.util.HashSet; -import java.util.Set; - -import static org.elasticsearch.test.SecuritySettingsSource.addSSLSettingsForNodePEMFiles; -import static org.elasticsearch.test.SecuritySettingsSource.addSSLSettingsForPEMFiles; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.is; - -/** - * Integration tests for the {@code ESNativeMigrateTool} - */ -public class ESNativeMigrateToolTests extends NativeRealmIntegTestCase { - - // Randomly use SSL (or not) - private static boolean useSSL; - - @BeforeClass - public static void setSSL() { - useSSL = randomBoolean(); - } - - @Override - protected boolean addMockHttpTransport() { - return false; // enable http - } - - @Override - public Settings nodeSettings(int nodeOrdinal) { - logger.info("--> use SSL? {}", useSSL); - Settings.Builder builder = Settings.builder() - .put(super.nodeSettings(nodeOrdinal)); - addSSLSettingsForNodePEMFiles(builder, "xpack.security.http.", true); - builder.put("xpack.security.http.ssl.enabled", useSSL); - return builder.build(); - } - - @Override - protected boolean transportSSLEnabled() { - return useSSL; - } - - @Override - protected boolean shouldSetReservedUserPasswords() { - return false; - } - - private Environment nodeEnvironment() throws Exception { - return internalCluster().getInstances(Environment.class).iterator().next(); - } - - public void testRetrieveUsers() throws Exception { - final Environment nodeEnvironment = nodeEnvironment(); - String home = Environment.PATH_HOME_SETTING.get(nodeEnvironment.settings()); - Path conf = nodeEnvironment.configFile(); - SecurityClient c = new SecurityClient(client()); - logger.error("--> creating users"); - int numToAdd = randomIntBetween(1,10); - Set addedUsers = new HashSet<>(numToAdd); - for (int i = 0; i < numToAdd; i++) { - String uname = randomAlphaOfLength(5); - c.preparePutUser(uname, "s3kirt".toCharArray(), getFastStoredHashAlgoForTests(), "role1", "user").get(); - addedUsers.add(uname); - } - logger.error("--> waiting for .security index"); - ensureGreen(RestrictedIndicesNames.SECURITY_MAIN_ALIAS); - - MockTerminal t = new MockTerminal(); - String username = nodeClientUsername(); - String password = new String(CharArrays.toUtf8Bytes(nodeClientPassword().getChars()), StandardCharsets.UTF_8); - String url = getHttpURL(); - ESNativeRealmMigrateTool.MigrateUserOrRoles muor = new ESNativeRealmMigrateTool.MigrateUserOrRoles(); - - Settings.Builder builder = Settings.builder() - .put("path.home", home) - .put("path.conf", conf.toString()) - .put("xpack.security.http.ssl.client_authentication", "none"); - addSSLSettingsForPEMFiles( - builder, - "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem", - "testnode", - "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt", - "xpack.security.http.", - Collections.singletonList("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt")); - Settings settings = builder.build(); - logger.error("--> retrieving users using URL: {}, home: {}", url, home); - - OptionParser parser = muor.getParser(); - OptionSet options = parser.parse("-u", username, "-p", password, "-U", url); - logger.info("--> options: {}", options.asMap()); - Set users = muor.getUsersThatExist(t, settings, new Environment(settings, conf), options); - logger.info("--> output: \n{}", t.getOutput()); - for (String u : addedUsers) { - assertThat("expected list to contain: " + u + ", real list: " + users, users.contains(u), is(true)); - } - } - - public void testRetrieveRoles() throws Exception { - final Environment nodeEnvironment = nodeEnvironment(); - String home = Environment.PATH_HOME_SETTING.get(nodeEnvironment.settings()); - Path conf = nodeEnvironment.configFile(); - SecurityClient c = new SecurityClient(client()); - logger.error("--> creating roles"); - int numToAdd = randomIntBetween(1,10); - Set addedRoles = new HashSet<>(numToAdd); - for (int i = 0; i < numToAdd; i++) { - String rname = randomAlphaOfLength(5); - c.preparePutRole(rname) - .cluster("all", "none") - .runAs("root", "nobody") - .addIndices(new String[] { "index" }, new String[] { "read" }, new String[] { "body", "title" }, null, - new BytesArray("{\"query\": {\"match_all\": {}}}"), randomBoolean()) - .get(); - addedRoles.add(rname); - } - logger.error("--> waiting for .security index"); - ensureGreen(RestrictedIndicesNames.SECURITY_MAIN_ALIAS); - - MockTerminal t = new MockTerminal(); - String username = nodeClientUsername(); - String password = new String(CharArrays.toUtf8Bytes(nodeClientPassword().getChars()), StandardCharsets.UTF_8); - String url = getHttpURL(); - ESNativeRealmMigrateTool.MigrateUserOrRoles muor = new ESNativeRealmMigrateTool.MigrateUserOrRoles(); - Settings.Builder builder = Settings.builder() - .put("path.home", home) - .put("xpack.security.http.ssl.client_authentication", "none"); - addSSLSettingsForPEMFiles(builder, - "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient.pem", - "testclient", - "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient.crt", - "xpack.security.http.", - Collections.singletonList("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt")); - Settings settings = builder.build(); - logger.error("--> retrieving roles using URL: {}, home: {}", url, home); - - OptionParser parser = muor.getParser(); - OptionSet options = parser.parse("-u", username, "-p", password, "-U", url); - Set roles = muor.getRolesThatExist(t, settings, new Environment(settings, conf), options); - logger.info("--> output: \n{}", t.getOutput()); - for (String r : addedRoles) { - assertThat("expected list to contain: " + r, roles.contains(r), is(true)); - } - } - - public void testMissingPasswordParameter() { - ESNativeRealmMigrateTool.MigrateUserOrRoles muor = new ESNativeRealmMigrateTool.MigrateUserOrRoles(); - - final OptionException ex = expectThrows(OptionException.class, - () -> muor.getParser().parse("-u", "elastic", "-U", "http://localhost:9200")); - - assertThat(ex.getMessage(), containsString("password")); - } -} diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeRealmMigrateToolTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeRealmMigrateToolTests.java deleted file mode 100644 index 212fd4a8dab42..0000000000000 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeRealmMigrateToolTests.java +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.security.authc.esnative; - -import joptsimple.OptionSet; -import org.apache.logging.log4j.Level; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.cli.Command; -import org.elasticsearch.cli.CommandTestCase; -import org.elasticsearch.cli.MockTerminal; -import org.elasticsearch.cli.Terminal.Verbosity; -import org.elasticsearch.cli.UserException; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.TestEnvironment; -import org.elasticsearch.test.SecuritySettingsSourceField; -import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; - -import java.io.FileNotFoundException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.isEmptyString; - -/** - * Unit tests for the {@code ESNativeRealmMigrateTool} - */ -public class ESNativeRealmMigrateToolTests extends CommandTestCase { - - @Override - protected Command newCommand() { - return new ESNativeRealmMigrateTool() { - @Override - protected MigrateUserOrRoles newMigrateUserOrRoles() { - return new MigrateUserOrRoles() { - - @Override - protected Environment createEnv(Map settings) throws UserException { - Settings.Builder builder = Settings.builder(); - settings.forEach((k, v) -> builder.put(k, v)); - return TestEnvironment.newEnvironment(builder.build()); - } - - }; - } - }; - } - - public void testUserJson() throws Exception { - assertThat(ESNativeRealmMigrateTool.MigrateUserOrRoles.createUserJson(Strings.EMPTY_ARRAY, "hash".toCharArray()), - equalTo("{\"password_hash\":\"hash\",\"roles\":[]}")); - assertThat(ESNativeRealmMigrateTool.MigrateUserOrRoles.createUserJson(new String[]{"role1", "role2"}, "hash".toCharArray()), - equalTo("{\"password_hash\":\"hash\",\"roles\":[\"role1\",\"role2\"]}")); - } - - public void testRoleJson() throws Exception { - RoleDescriptor.IndicesPrivileges ip = RoleDescriptor.IndicesPrivileges.builder() - .indices(new String[]{"i1", "i2", "i3"}) - .privileges(new String[]{"all"}) - .grantedFields("body") - .build(); - RoleDescriptor.IndicesPrivileges[] ips = new RoleDescriptor.IndicesPrivileges[1]; - ips[0] = ip; - String[] cluster = Strings.EMPTY_ARRAY; - String[] runAs = Strings.EMPTY_ARRAY; - RoleDescriptor rd = new RoleDescriptor("rolename", cluster, ips, runAs); - assertThat(ESNativeRealmMigrateTool.MigrateUserOrRoles.createRoleJson(rd), - equalTo("{\"cluster\":[]," + - "\"indices\":[{\"names\":[\"i1\",\"i2\",\"i3\"]," + - "\"privileges\":[\"all\"],\"field_security\":{\"grant\":[\"body\"]}," + - "\"allow_restricted_indices\":false}]," + - "\"applications\":[]," + - "\"run_as\":[],\"metadata\":{},\"type\":\"role\"}")); - } - - public void testTerminalLogger() throws Exception { - Logger terminalLogger = ESNativeRealmMigrateTool.getTerminalLogger(terminal); - assertThat(terminal.getOutput(), isEmptyString()); - - // only error and fatal gets logged at normal verbosity - terminal.setVerbosity(Verbosity.NORMAL); - List nonLoggingLevels = new ArrayList<>(Arrays.asList(Level.values())); - nonLoggingLevels.removeAll(Arrays.asList(Level.ERROR, Level.FATAL)); - for (Level level : nonLoggingLevels) { - terminalLogger.log(level, "this level should not log " + level.name()); - assertThat(terminal.getOutput(), isEmptyString()); - } - - terminalLogger.log(Level.ERROR, "logging an error"); - assertEquals("logging an error\n", terminal.getOutput()); - terminal.reset(); - assertThat(terminal.getOutput(), isEmptyString()); - - terminalLogger.log(Level.FATAL, "logging a fatal message"); - assertEquals("logging a fatal message\n", terminal.getOutput()); - terminal.reset(); - assertThat(terminal.getOutput(), isEmptyString()); - - // everything will get logged at verbose! - terminal.setVerbosity(Verbosity.VERBOSE); - List loggingLevels = new ArrayList<>(Arrays.asList(Level.values())); - loggingLevels.remove(Level.OFF); - for (Level level : loggingLevels) { - terminalLogger.log(level, "this level should log " + level.name()); - assertEquals("this level should log " + level.name() + "\n", terminal.getOutput()); - terminal.reset(); - assertThat(terminal.getOutput(), isEmptyString()); - } - } - - public void testMissingFiles() throws Exception { - Path homeDir = createTempDir(); - Path confDir = homeDir.resolve("config"); - Path xpackConfDir = confDir; - Files.createDirectories(xpackConfDir); - - ESNativeRealmMigrateTool.MigrateUserOrRoles muor = new ESNativeRealmMigrateTool.MigrateUserOrRoles(); - - OptionSet options = muor.getParser().parse("-u", "elastic", "-p", SecuritySettingsSourceField.TEST_PASSWORD, - "-U", "http://localhost:9200"); - Settings settings = Settings.builder().put("path.home", homeDir).build(); - Environment environment = new Environment(settings, confDir); - - MockTerminal mockTerminal = new MockTerminal(); - - FileNotFoundException fnfe = expectThrows(FileNotFoundException.class, - () -> muor.importUsers(mockTerminal, environment, options)); - assertThat(fnfe.getMessage(), containsString("users file")); - - Files.createFile(xpackConfDir.resolve("users")); - fnfe = expectThrows(FileNotFoundException.class, - () -> muor.importUsers(mockTerminal, environment, options)); - assertThat(fnfe.getMessage(), containsString("users_roles file")); - - fnfe = expectThrows(FileNotFoundException.class, - () -> muor.importRoles(mockTerminal, environment, options)); - assertThat(fnfe.getMessage(), containsString("roles.yml file")); - } -} diff --git a/x-pack/qa/security-migrate-tests/src/test/java/org/elasticsearch/xpack/security/MigrateToolIT.java b/x-pack/qa/security-migrate-tests/src/test/java/org/elasticsearch/xpack/security/MigrateToolIT.java deleted file mode 100644 index 3581bf2fda7fd..0000000000000 --- a/x-pack/qa/security-migrate-tests/src/test/java/org/elasticsearch/xpack/security/MigrateToolIT.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.security; - -import joptsimple.OptionParser; -import joptsimple.OptionSet; - -import org.elasticsearch.cli.MockTerminal; -import org.elasticsearch.client.Client; -import org.elasticsearch.client.Requests; -import org.elasticsearch.common.Priority; -import org.elasticsearch.common.io.PathUtils; -import org.elasticsearch.common.settings.SecureString; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.env.Environment; -import org.elasticsearch.xpack.core.security.action.role.GetRolesResponse; -import org.elasticsearch.xpack.core.security.action.user.GetUsersResponse; -import org.elasticsearch.xpack.core.security.action.user.PutUserResponse; -import org.elasticsearch.xpack.core.security.authc.support.Hasher; -import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; -import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissions; -import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsDefinition; -import org.elasticsearch.xpack.core.security.client.SecurityClient; -import org.elasticsearch.xpack.core.security.user.User; -import org.elasticsearch.xpack.security.authc.esnative.ESNativeRealmMigrateTool; -import org.junit.Before; - -import java.nio.file.Path; -import java.util.Arrays; -import java.util.Collections; - -import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; -import static org.hamcrest.Matchers.containsString; - -/** - * Integration tests for the {@code elasticsearch-migrate} shell command - */ -public class MigrateToolIT extends MigrateToolTestCase { - - @Before - public void setupUpTest() throws Exception { - Client client = getClient(); - SecurityClient c = new SecurityClient(client); - - // Add an existing user so the tool will skip it - PutUserResponse pur = c.preparePutUser("existing", "s3kirt".toCharArray(), Hasher.BCRYPT, "role1", "user").get(); - assertTrue(pur.created()); - } - - public void testRunMigrateTool() throws Exception { - final String testConfigDir = System.getProperty("tests.config.dir"); - logger.info("--> CONF: {}", testConfigDir); - final Path configPath = PathUtils.get(testConfigDir); - Settings settings = Settings.builder().put("path.home", configPath.getParent()).build(); - // Cluster should already be up - String url = "http://" + getHttpURL(); - logger.info("--> using URL: {}", url); - MockTerminal t = new MockTerminal(); - ESNativeRealmMigrateTool.MigrateUserOrRoles muor = new ESNativeRealmMigrateTool.MigrateUserOrRoles(); - OptionParser parser = muor.getParser(); - - OptionSet options = parser.parse("-u", "test_admin", "-p", "x-pack-test-password", "-U", url); - muor.execute(t, options, new Environment(settings, configPath)); - - logger.info("--> output:\n{}", t.getOutput()); - - Client client = getClient(); - SecurityClient c = new SecurityClient(client); - - // Check that the migrated user can be retrieved - GetUsersResponse resp = c.prepareGetUsers("bob").get(); - assertTrue("user 'bob' should exist", resp.hasUsers()); - User bob = resp.users()[0]; - assertEquals(bob.principal(), "bob"); - assertArrayEquals(bob.roles(), new String[]{"actual_role"}); - - // Make sure the existing user did not change - resp = c.prepareGetUsers("existing").get(); - assertTrue("user should exist", resp.hasUsers()); - User existing = resp.users()[0]; - assertEquals(existing.principal(), "existing"); - assertArrayEquals(existing.roles(), new String[]{"role1", "user"}); - - // Make sure the "actual_role" made it in and is correct - GetRolesResponse roleResp = c.prepareGetRoles().names("actual_role").get(); - assertTrue("role should exist", roleResp.hasRoles()); - RoleDescriptor rd = roleResp.roles()[0]; - assertNotNull(rd); - assertEquals(rd.getName(), "actual_role"); - assertArrayEquals(rd.getClusterPrivileges(), new String[]{"monitor"}); - assertArrayEquals(rd.getRunAs(), new String[]{"joe"}); - RoleDescriptor.IndicesPrivileges[] ips = rd.getIndicesPrivileges(); - assertEquals(ips.length, 2); - for (RoleDescriptor.IndicesPrivileges ip : ips) { - final FieldPermissions fieldPermissions = new FieldPermissions( - new FieldPermissionsDefinition(ip.getGrantedFields(), ip.getDeniedFields())); - if (Arrays.equals(ip.getIndices(), new String[]{"index1", "index2"})) { - assertArrayEquals(ip.getPrivileges(), new String[]{"read", "write", "create_index", "indices:admin/refresh"}); - assertTrue(fieldPermissions.hasFieldLevelSecurity()); - assertTrue(fieldPermissions.grantsAccessTo("bar")); - assertTrue(fieldPermissions.grantsAccessTo("foo")); - assertNotNull(ip.getQuery()); - assertThat(ip.getQuery().iterator().next().utf8ToString(), - containsString("{\"bool\":{\"must_not\":{\"match\":{\"hidden\":true}}}}")); - } else { - assertArrayEquals(ip.getIndices(), new String[]{"*"}); - assertArrayEquals(ip.getPrivileges(), new String[]{"read"}); - assertFalse(fieldPermissions.hasFieldLevelSecurity()); - assertNull(ip.getQuery()); - } - } - - // Check that bob can access the things the "actual_role" says he can - String token = basicAuthHeaderValue("bob", new SecureString("x-pack-test-password".toCharArray())); - // Create "index1" index and try to search from it as "bob" - client.filterWithHeader(Collections.singletonMap("Authorization", token)).admin().indices().prepareCreate("index1").get(); - // Wait for the index to be ready so it doesn't fail if no shards are initialized - client.admin().cluster().health(Requests.clusterHealthRequest("index1") - .timeout(TimeValue.timeValueSeconds(30)) - .waitForYellowStatus() - .waitForEvents(Priority.LANGUID) - .waitForNoRelocatingShards(true)) - .actionGet(); - client.filterWithHeader(Collections.singletonMap("Authorization", token)).prepareSearch("index1").get(); - } -} diff --git a/x-pack/qa/security-migrate-tests/src/test/java/org/elasticsearch/xpack/security/MigrateToolTestCase.java b/x-pack/qa/security-migrate-tests/src/test/java/org/elasticsearch/xpack/security/MigrateToolTestCase.java deleted file mode 100644 index 0111aeff4cca2..0000000000000 --- a/x-pack/qa/security-migrate-tests/src/test/java/org/elasticsearch/xpack/security/MigrateToolTestCase.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.security; - -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.LogManager; -import org.apache.lucene.util.LuceneTestCase; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.client.transport.TransportClient; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.xpack.client.PreBuiltXPackTransportClient; -import org.elasticsearch.xpack.core.security.SecurityField; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; - -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.nio.file.Path; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.hamcrest.Matchers.notNullValue; - -/** - * {@link MigrateToolTestCase} is an abstract base class to run integration - * tests against an external Elasticsearch Cluster. - *

- * You can define a list of transport addresses from where you can reach your cluster - * by setting "tests.cluster" system property. It defaults to "localhost:9300". - *

- * All tests can be run from maven using mvn install as maven will start an external cluster first. - *

- * If you want to debug this module from your IDE, then start an external cluster by yourself - * then run JUnit. If you changed the default port, set "tests.cluster=localhost:PORT" when running - * your test. - */ -@LuceneTestCase.SuppressSysoutChecks(bugUrl = "we log a lot on purpose") -public abstract class MigrateToolTestCase extends LuceneTestCase { - - /** - * Key used to eventually switch to using an external cluster and provide its transport addresses - */ - public static final String TESTS_CLUSTER = "tests.cluster"; - - /** - * Key used to eventually switch to using an external cluster and provide its transport addresses - */ - public static final String TESTS_HTTP_CLUSTER = "tests.rest.cluster"; - - /** - * Defaults to localhost:9300 - */ - public static final String TESTS_CLUSTER_DEFAULT = "localhost:9300"; - - protected static final Logger logger = LogManager.getLogger(MigrateToolTestCase.class); - - private static final AtomicInteger counter = new AtomicInteger(); - private static Client client; - private static String clusterAddresses; - private static String clusterHttpAddresses; - - private static Client startClient(Path tempDir, TransportAddress... transportAddresses) { - logger.info("--> Starting Elasticsearch Java TransportClient {}, {}", transportAddresses, tempDir); - - Settings clientSettings = Settings.builder() - .put("cluster.name", "qa_migrate_tests_" + counter.getAndIncrement()) - .put("client.transport.ignore_cluster_name", true) - .put("path.home", tempDir) - .put(SecurityField.USER_SETTING.getKey(), "transport_user:x-pack-test-password") - .build(); - - TransportClient client = new PreBuiltXPackTransportClient(clientSettings).addTransportAddresses(transportAddresses); - Exception clientException = null; - try { - logger.info("--> Elasticsearch Java TransportClient started"); - ClusterHealthResponse health = client.admin().cluster().prepareHealth().get(); - logger.info("--> connected to [{}] cluster which is running [{}] node(s).", - health.getClusterName(), health.getNumberOfNodes()); - } catch (Exception e) { - clientException = e; - } - - assumeNoException("Sounds like your cluster is not running at " + clusterAddresses, clientException); - - return client; - } - - private static Client startClient() throws UnknownHostException { - String[] stringAddresses = clusterAddresses.split(","); - TransportAddress[] transportAddresses = new TransportAddress[stringAddresses.length]; - int i = 0; - for (String stringAddress : stringAddresses) { - int lastColon = stringAddress.lastIndexOf(":"); - if (lastColon == -1) { - throw new IllegalArgumentException("address [" + clusterAddresses + "] not valid"); - } - String ip = stringAddress.substring(0, lastColon); - String port = stringAddress.substring(lastColon + 1); - try { - transportAddresses[i++] = new TransportAddress(InetAddress.getByName(ip), Integer.valueOf(port)); - } catch (NumberFormatException e) { - throw new IllegalArgumentException("port is not valid, expected number but was [" + port + "]"); - } - } - return startClient(createTempDir(), transportAddresses); - } - - public static Client getClient() { - if (client == null) { - try { - client = startClient(); - } catch (UnknownHostException e) { - logger.error("could not start the client", e); - } - assertThat(client, notNullValue()); - } - return client; - } - - public static String getHttpURL() { - return clusterHttpAddresses; - } - - @BeforeClass - public static void initializeSettings() throws UnknownHostException { - clusterAddresses = System.getProperty(TESTS_CLUSTER); - clusterHttpAddresses = System.getProperty(TESTS_HTTP_CLUSTER); - if (clusterAddresses == null || clusterAddresses.isEmpty()) { - throw new UnknownHostException("unable to get a cluster address"); - } - } - - @AfterClass - public static void stopTransportClient() { - if (client != null) { - client.close(); - client = null; - } - } - - @Before - public void defineIndexName() { - doClean(); - } - - @After - public void cleanIndex() { - doClean(); - } - - private void doClean() { - if (client != null) { - try { - client.admin().indices().prepareDelete("_all").get(); - } catch (Exception e) { - // We ignore this cleanup exception - } - } - } -} From aca47a0a07ab7d839fd828e374c088983a7ee581 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 16 May 2019 10:40:16 -0400 Subject: [PATCH 2/3] remove qa test project --- x-pack/qa/security-migrate-tests/build.gradle | 43 ------------------- x-pack/qa/security-migrate-tests/roles.yml | 22 ---------- 2 files changed, 65 deletions(-) delete mode 100644 x-pack/qa/security-migrate-tests/build.gradle delete mode 100644 x-pack/qa/security-migrate-tests/roles.yml diff --git a/x-pack/qa/security-migrate-tests/build.gradle b/x-pack/qa/security-migrate-tests/build.gradle deleted file mode 100644 index 1851f0e21b027..0000000000000 --- a/x-pack/qa/security-migrate-tests/build.gradle +++ /dev/null @@ -1,43 +0,0 @@ -apply plugin: 'elasticsearch.standalone-rest-test' -apply plugin: 'elasticsearch.rest-test' - -dependencies { - testCompile "org.elasticsearch.plugin:x-pack-core:${version}" - testCompile project(path: xpackModule('security'), configuration: 'runtime') - testCompile project(path: xpackProject('transport-client').path, configuration: 'runtime') -} - -integTestCluster { - setting 'xpack.security.enabled', 'true' - setting 'xpack.license.self_generated.type', 'trial' - extraConfigFile 'roles.yml', 'roles.yml' - [ - test_admin: 'superuser', - transport_user: 'superuser', - existing: 'superuser', - bob: 'actual_role' - ].each { String user, String role -> - setupCommand 'setupUser#' + user, - 'bin/elasticsearch-users', 'useradd', user, '-p', 'x-pack-test-password', '-r', role - } - waitCondition = { node, ant -> - File tmpFile = new File(node.cwd, 'wait.success') - ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow", - dest: tmpFile.toString(), - username: 'test_admin', - password: 'x-pack-test-password', - ignoreerrors: true, - retries: 10) - return tmpFile.exists() - } - // TODO: systemProperty('tests.cluster', "${-> cluster.transportPortURI }") when migerating to testclusters -} - -testingConventions { - naming.clear() - naming { - IT { - baseClass 'org.elasticsearch.xpack.security.MigrateToolTestCase' - } - } -} diff --git a/x-pack/qa/security-migrate-tests/roles.yml b/x-pack/qa/security-migrate-tests/roles.yml deleted file mode 100644 index 6e997383f8a5a..0000000000000 --- a/x-pack/qa/security-migrate-tests/roles.yml +++ /dev/null @@ -1,22 +0,0 @@ -# A role that has all sorts of configuration: -# - it can monitor the cluster -# - for index1 and index2 it can do CRUD things and refresh -# - for other indices it has search-only privileges -actual_role: - run_as: [ "joe" ] - cluster: - - monitor - indices: - - names: [ "index1", "index2" ] - privileges: [ "read", "write", "create_index", "indices:admin/refresh" ] - field_security: - grant: - - foo - - bar - query: - bool: - must_not: - match: - hidden: true - - names: "*" - privileges: [ "read" ] From a836dc6f01c37bab7156fbb2b64fe68f18338155 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Fri, 17 May 2019 08:00:56 -0400 Subject: [PATCH 3/3] remove from packaging tests --- .../src/main/java/org/elasticsearch/packaging/util/Archives.java | 1 - .../src/main/java/org/elasticsearch/packaging/util/Packages.java | 1 - x-pack/qa/vagrant/src/test/resources/packaging/utils/xpack.bash | 1 - 3 files changed, 3 deletions(-) diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java index e557b47fb8912..2eb3a288fbcc2 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java @@ -222,7 +222,6 @@ private static void verifyDefaultInstallation(Installation es, Distribution dist "elasticsearch-certgen", "elasticsearch-certutil", "elasticsearch-croneval", - "elasticsearch-migrate", "elasticsearch-saml-metadata", "elasticsearch-setup-passwords", "elasticsearch-sql-cli", diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Packages.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Packages.java index 70ac89dc3b7f5..4d528b96c32e9 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Packages.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Packages.java @@ -244,7 +244,6 @@ private static void verifyDefaultInstallation(Installation es) { "elasticsearch-certgen", "elasticsearch-certutil", "elasticsearch-croneval", - "elasticsearch-migrate", "elasticsearch-saml-metadata", "elasticsearch-setup-passwords", "elasticsearch-sql-cli", diff --git a/x-pack/qa/vagrant/src/test/resources/packaging/utils/xpack.bash b/x-pack/qa/vagrant/src/test/resources/packaging/utils/xpack.bash index c267744194a1c..bafe7d9342f0e 100644 --- a/x-pack/qa/vagrant/src/test/resources/packaging/utils/xpack.bash +++ b/x-pack/qa/vagrant/src/test/resources/packaging/utils/xpack.bash @@ -17,7 +17,6 @@ verify_xpack_installation() { 'elasticsearch-certgen' 'elasticsearch-certutil' 'elasticsearch-croneval' - 'elasticsearch-migrate' 'elasticsearch-saml-metadata' 'elasticsearch-setup-passwords' 'elasticsearch-sql-cli'