From 9493da40313f31a426266a71e1aae8702571daf4 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Tue, 1 Jan 2019 11:42:31 +0200 Subject: [PATCH 01/19] Fork out the security tokens template --- .../security-tokens-index-template.json | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 x-pack/plugin/core/src/main/resources/security-tokens-index-template.json diff --git a/x-pack/plugin/core/src/main/resources/security-tokens-index-template.json b/x-pack/plugin/core/src/main/resources/security-tokens-index-template.json new file mode 100644 index 0000000000000..88e4d914e5c5e --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/security-tokens-index-template.json @@ -0,0 +1,85 @@ +{ + "index_patterns" : [ ".security-tokens-*" ], + "order" : 1000, + "settings" : { + "number_of_shards" : 1, + "number_of_replicas" : 0, + "auto_expand_replicas" : "0-1", + "index.priority": 1000, + "index.format": 7 + }, + "mappings" : { + "doc" : { + "_meta": { + "security-version": "${security.template.version}" + }, + "dynamic" : "strict", + "properties" : { + "doc_type" : { + "type" : "keyword" + }, + "refresh_token" : { + "type" : "object", + "properties" : { + "token" : { + "type" : "keyword" + }, + "refreshed" : { + "type" : "boolean" + }, + "invalidated" : { + "type" : "boolean" + }, + "client" : { + "type" : "object", + "properties" : { + "type" : { + "type" : "keyword" + }, + "user" : { + "type" : "keyword" + }, + "realm" : { + "type" : "keyword" + } + } + } + } + }, + "access_token" : { + "type" : "object", + "properties" : { + "user_token" : { + "type" : "object", + "properties" : { + "id" : { + "type" : "keyword" + }, + "expiration_time" : { + "type" : "date", + "format" : "epoch_millis" + }, + "version" : { + "type" : "integer" + }, + "metadata" : { + "type" : "object", + "dynamic" : true + }, + "authentication" : { + "type" : "binary" + } + } + }, + "invalidated" : { + "type" : "boolean" + }, + "realm" : { + "type" : "keyword" + } + } + } + } + } + } +} From c4d8a0a1a9fc876a644a94af4f957af31da761d8 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Tue, 1 Jan 2019 11:43:27 +0200 Subject: [PATCH 02/19] No templates in SecurityIndexManager --- .../xpack/security/support/SecurityIndexManager.java | 9 +-------- .../security/support/SecurityIndexManagerTests.java | 10 ++++------ 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java index acd4f1c480500..58f33c4277cce 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java @@ -63,7 +63,7 @@ import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; /** - * Manages the lifecycle of a single index, its template, mapping and and data upgrades/migrations. + * Manages the lifecycle of a single index, its mapping and and data upgrades/migrations. */ public class SecurityIndexManager implements ClusterStateListener { @@ -198,13 +198,6 @@ private IndexRoutingTable getIndexRoutingTable(ClusterState clusterState) { } } - public static boolean checkTemplateExistsAndVersionMatches( - String templateName, ClusterState state, Logger logger, Predicate predicate) { - - return TemplateUtils.checkTemplateExistsAndVersionMatches(templateName, SECURITY_VERSION_STRING, - state, logger, predicate); - } - private boolean checkIndexMappingUpToDate(ClusterState clusterState) { return checkIndexMappingVersionMatches(clusterState, Version.CURRENT::equals); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java index 0741d1c04e995..4fdb3ce6c5913 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java @@ -346,12 +346,10 @@ public void testIndexTemplateVersionMatching() throws Exception { ClusterState.Builder clusterStateBuilder = createClusterStateWithTemplate(templateString); final ClusterState clusterState = clusterStateBuilder.build(); - assertTrue(SecurityIndexManager.checkTemplateExistsAndVersionMatches( - SecurityIndexManager.SECURITY_TEMPLATE_NAME, clusterState, logger, - Version.V_6_0_0::before)); - assertFalse(SecurityIndexManager.checkTemplateExistsAndVersionMatches( - SecurityIndexManager.SECURITY_TEMPLATE_NAME, clusterState, logger, - Version.V_6_0_0::after)); + assertTrue(TemplateUtils.checkTemplateExistsAndVersionMatches(SecurityIndexManager.SECURITY_TEMPLATE_NAME, + SecurityIndexManager.SECURITY_VERSION_STRING, clusterState, logger, Version.V_6_0_0::before)); + assertFalse(TemplateUtils.checkTemplateExistsAndVersionMatches(SecurityIndexManager.SECURITY_TEMPLATE_NAME, + SecurityIndexManager.SECURITY_VERSION_STRING, clusterState, logger, Version.V_6_0_0::after)); } public void testUpToDateMappingsAreIdentifiedAsUpToDate() throws IOException { From 903dbb71ca6c98d4d165b94aa3870a3a4bcd29c4 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Tue, 1 Jan 2019 12:28:42 +0200 Subject: [PATCH 03/19] Remove extraneous method from SecurityManager --- .../security/support/SecurityIndexManager.java | 15 ++++----------- .../elasticsearch/test/SecurityIntegTestCase.java | 9 +++++---- .../support/SecurityIndexManagerTests.java | 2 +- 3 files changed, 10 insertions(+), 16 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java index 58f33c4277cce..ca82d043d3bfa 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java @@ -199,19 +199,12 @@ private IndexRoutingTable getIndexRoutingTable(ClusterState clusterState) { } private boolean checkIndexMappingUpToDate(ClusterState clusterState) { - return checkIndexMappingVersionMatches(clusterState, Version.CURRENT::equals); + return checkIndexMappingVersionMatches(indexName, clusterState, Version.CURRENT::equals); } - private boolean checkIndexMappingVersionMatches(ClusterState clusterState, - Predicate predicate) { - return checkIndexMappingVersionMatches(indexName, clusterState, LOGGER, predicate); - } - - public static boolean checkIndexMappingVersionMatches(String indexName, - ClusterState clusterState, Logger logger, - Predicate predicate) { - return loadIndexMappingVersions(indexName, clusterState, logger) - .stream().allMatch(predicate); + // public and static for testing + public static boolean checkIndexMappingVersionMatches(String indexName, ClusterState clusterState, Predicate predicate) { + return loadIndexMappingVersions(indexName, clusterState, LOGGER).stream().allMatch(predicate); } private Version oldestIndexMappingVersion(ClusterState clusterState) { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java index c3e3bddf10e97..5d227de626484 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java @@ -489,10 +489,11 @@ public void assertSecurityIndexActive(TestCluster testCluster) throws Exception ClusterState clusterState = client.admin().cluster().prepareState().setLocal(true).get().getState(); assertFalse(clusterState.blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)); XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint().startObject(); - assertTrue("security index mapping not sufficient to read:\n" + - Strings.toString(clusterState.toXContent(builder, ToXContent.EMPTY_PARAMS).endObject()), - SecurityIndexManager.checkIndexMappingVersionMatches(SECURITY_INDEX_NAME, clusterState, logger, - Version.CURRENT.minimumIndexCompatibilityVersion()::onOrBefore)); + assertTrue( + "security index mapping not sufficient to read:\n" + + Strings.toString(clusterState.toXContent(builder, ToXContent.EMPTY_PARAMS).endObject()), + SecurityIndexManager.checkIndexMappingVersionMatches(SECURITY_INDEX_NAME, clusterState, + Version.CURRENT.minimumIndexCompatibilityVersion()::onOrBefore)); Index securityIndex = resolveSecurityIndex(clusterState.metaData()); if (securityIndex != null) { IndexRoutingTable indexRoutingTable = clusterState.routingTable().index(securityIndex); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java index 4fdb3ce6c5913..0caeff4ddf64c 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java @@ -328,7 +328,7 @@ public void testMissingVersionMappingThrowsError() throws IOException { ClusterState.Builder clusterStateBuilder = createClusterStateWithMappingAndTemplate(templateString); final ClusterState clusterState = clusterStateBuilder.build(); IllegalStateException exception = expectThrows(IllegalStateException.class, - () -> SecurityIndexManager.checkIndexMappingVersionMatches(SECURITY_INDEX_NAME, clusterState, logger, Version.CURRENT::equals)); + () -> SecurityIndexManager.checkIndexMappingVersionMatches(SECURITY_INDEX_NAME, clusterState, Version.CURRENT::equals)); assertEquals("Cannot read security-version string in index " + SECURITY_INDEX_NAME, exception.getMessage()); } From 748b8fb0be27c3f9d6d6c23e5aea0812bfaff96f Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Tue, 1 Jan 2019 16:16:25 +0200 Subject: [PATCH 04/19] SecurityIndexManager independent of static --- .../xpack/security/Security.java | 6 +- .../security/authc/ExpiredTokenRemover.java | 2 +- .../xpack/security/authc/TokenService.java | 18 ++--- .../authc/esnative/NativeUsersStore.java | 26 +++---- .../mapper/NativeRoleMappingStore.java | 14 ++-- .../security/authz/AuthorizationService.java | 2 +- .../authz/store/NativePrivilegeStore.java | 10 +-- .../authz/store/NativeRolesStore.java | 18 ++--- .../support/SecurityIndexManager.java | 40 +++++++---- .../integration/ClearRolesCacheTests.java | 2 +- .../test/SecurityIntegTestCase.java | 8 +-- .../xpack/security/SecurityTests.java | 8 +-- .../security/authc/TokenAuthIntegTests.java | 8 +-- .../esnative/ESNativeMigrateToolTests.java | 4 +- .../authc/esnative/NativeRealmIntegTests.java | 18 ++--- .../authc/esnative/NativeUsersStoreTests.java | 6 +- .../authz/AuthorizationServiceTests.java | 68 +++++++++---------- .../authz/AuthorizedIndicesTests.java | 6 +- .../authz/IndicesAndAliasesResolverTests.java | 16 ++--- .../store/NativePrivilegeStoreTests.java | 16 ++--- .../authz/store/NativeRolesStoreTests.java | 7 +- .../support/SecurityIndexManagerTests.java | 12 ++-- .../security/test/SecurityTestUtils.java | 4 +- .../xpack/security/user/XPackUserTests.java | 2 +- 24 files changed, 165 insertions(+), 156 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index ad9f1d7aa948c..062b804f55d65 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -253,7 +253,7 @@ import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_FORMAT_SETTING; import static org.elasticsearch.xpack.core.XPackSettings.HTTP_SSL_ENABLED; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.INTERNAL_INDEX_FORMAT; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_ALIAS_NAME; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_TEMPLATE_NAME; public class Security extends Plugin implements ActionPlugin, IngestPlugin, NetworkPlugin, ClusterPlugin, @@ -428,7 +428,7 @@ Collection createComponents(Client client, ThreadPool threadPool, Cluste components.add(auditTrailService); this.auditTrailService.set(auditTrailService); - securityIndex.set(new SecurityIndexManager(client, SecurityIndexManager.SECURITY_INDEX_NAME, clusterService)); + securityIndex.set(SecurityIndexManager.buildSecurityIndexManager(client, clusterService)); final TokenService tokenService = new TokenService(settings, Clock.systemUTC(), client, securityIndex.get(), clusterService); this.tokenService.set(tokenService); @@ -1076,7 +1076,7 @@ static final class ValidateUpgradedSecurityIndex implements BiConsumer listener) throws IOExce securityIndex.checkIndexVersionThenExecute( ex -> listener.onFailure(traceLog("prepare security index", tokenId, ex)), () -> { - final GetRequest getRequest = client.prepareGet(SecurityIndexManager.SECURITY_INDEX_NAME, TYPE, + final GetRequest getRequest = client.prepareGet(SecurityIndexManager.SECURITY_ALIAS_NAME, TYPE, getTokenDocumentId(tokenId)).request(); Consumer onFailure = ex -> listener.onFailure(traceLog("decode token", tokenId, ex)); executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, getRequest, @@ -584,7 +584,7 @@ private void indexInvalidation(Collection tokenIds, ActionListener onFailure = ex -> listener.onFailure(traceLog("refresh token", tokenDocId, ex)); - GetRequest getRequest = client.prepareGet(SecurityIndexManager.SECURITY_INDEX_NAME, TYPE, tokenDocId).request(); + GetRequest getRequest = client.prepareGet(SecurityIndexManager.SECURITY_ALIAS_NAME, TYPE, tokenDocId).request(); executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, getRequest, ActionListener.wrap(response -> { if (response.isExists()) { @@ -745,7 +745,7 @@ private void innerRefresh(String tokenDocId, Authentication userAuth, ActionList in.setVersion(authVersion); Authentication authentication = new Authentication(in); UpdateRequest updateRequest = - client.prepareUpdate(SecurityIndexManager.SECURITY_INDEX_NAME, TYPE, tokenDocId) + client.prepareUpdate(SecurityIndexManager.SECURITY_ALIAS_NAME, TYPE, tokenDocId) .setVersion(response.getVersion()) .setDoc("refresh_token", Collections.singletonMap("refreshed", true)) .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL) @@ -870,7 +870,7 @@ public void findActiveTokensForRealm(String realmName, ActionListener listener.onResponse(null); } else { securityIndex.checkIndexVersionThenExecute(listener::onFailure, () -> { - final GetRequest getRequest = client.prepareGet(SecurityIndexManager.SECURITY_INDEX_NAME, TYPE, + final GetRequest getRequest = client.prepareGet(SecurityIndexManager.SECURITY_ALIAS_NAME, TYPE, getTokenDocumentId(userToken)).request(); Consumer onFailure = ex -> listener.onFailure(traceLog("check token state", userToken.getId(), ex)); executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, getRequest, diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java index 47c32489ae1f4..6d5b0a0c058bc 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java @@ -66,7 +66,7 @@ import static org.elasticsearch.xpack.core.ClientHelper.SECURITY_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; import static org.elasticsearch.xpack.core.ClientHelper.stashWithOrigin; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_ALIAS_NAME; /** * NativeUsersStore is a store for users that reads from an Elasticsearch index. This store is responsible for fetching the full @@ -147,7 +147,7 @@ public void getUsers(String[] userNames, final ActionListener> } final Supplier supplier = client.threadPool().getThreadContext().newRestorableContext(false); try (ThreadContext.StoredContext ignore = stashWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN)) { - SearchRequest request = client.prepareSearch(SECURITY_INDEX_NAME) + SearchRequest request = client.prepareSearch(SECURITY_ALIAS_NAME) .setScroll(DEFAULT_KEEPALIVE_SETTING.get(settings)) .setQuery(query) .setSize(1000) @@ -172,7 +172,7 @@ void getUserCount(final ActionListener listener) { } else { securityIndex.checkIndexVersionThenExecute(listener::onFailure, () -> executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, - client.prepareSearch(SECURITY_INDEX_NAME) + client.prepareSearch(SECURITY_ALIAS_NAME) .setQuery(QueryBuilders.termQuery(Fields.TYPE.getPreferredName(), USER_DOC_TYPE)) .setSize(0) .request(), @@ -205,7 +205,7 @@ private void getUserAndPassword(final String user, final ActionListener executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, - client.prepareGet(SECURITY_INDEX_NAME, + client.prepareGet(SECURITY_ALIAS_NAME, INDEX_TYPE, getIdForUser(USER_DOC_TYPE, user)).request(), new ActionListener() { @Override @@ -246,7 +246,7 @@ public void changePassword(final ChangePasswordRequest request, final ActionList securityIndex.prepareIndexIfNeededThenExecute(listener::onFailure, () -> { executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, - client.prepareUpdate(SECURITY_INDEX_NAME, INDEX_TYPE, getIdForUser(docType, username)) + client.prepareUpdate(SECURITY_ALIAS_NAME, INDEX_TYPE, getIdForUser(docType, username)) .setDoc(Requests.INDEX_CONTENT_TYPE, Fields.PASSWORD.getPreferredName(), String.valueOf(request.passwordHash())) .setRefreshPolicy(request.getRefreshPolicy()).request(), @@ -284,7 +284,7 @@ public void onFailure(Exception e) { private void createReservedUser(String username, char[] passwordHash, RefreshPolicy refresh, ActionListener listener) { securityIndex.prepareIndexIfNeededThenExecute(listener::onFailure, () -> { executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, - client.prepareIndex(SECURITY_INDEX_NAME, INDEX_TYPE, + client.prepareIndex(SECURITY_ALIAS_NAME, INDEX_TYPE, getIdForUser(RESERVED_USER_TYPE, username)) .setSource(Fields.PASSWORD.getPreferredName(), String.valueOf(passwordHash), Fields.ENABLED.getPreferredName(), true, @@ -326,7 +326,7 @@ private void updateUserWithoutPassword(final PutUserRequest putUserRequest, fina // We must have an existing document securityIndex.prepareIndexIfNeededThenExecute(listener::onFailure, () -> { executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, - client.prepareUpdate(SECURITY_INDEX_NAME, INDEX_TYPE, + client.prepareUpdate(SECURITY_ALIAS_NAME, INDEX_TYPE, getIdForUser(USER_DOC_TYPE, putUserRequest.username())) .setDoc(Requests.INDEX_CONTENT_TYPE, Fields.USERNAME.getPreferredName(), putUserRequest.username(), @@ -371,7 +371,7 @@ private void indexUser(final PutUserRequest putUserRequest, final ActionListener assert putUserRequest.passwordHash() != null; securityIndex.prepareIndexIfNeededThenExecute(listener::onFailure, () -> { executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, - client.prepareIndex(SECURITY_INDEX_NAME, INDEX_TYPE, + client.prepareIndex(SECURITY_ALIAS_NAME, INDEX_TYPE, getIdForUser(USER_DOC_TYPE, putUserRequest.username())) .setSource(Fields.USERNAME.getPreferredName(), putUserRequest.username(), Fields.PASSWORD.getPreferredName(), String.valueOf(putUserRequest.passwordHash()), @@ -415,7 +415,7 @@ private void setRegularUserEnabled(final String username, final boolean enabled, final ActionListener listener) { securityIndex.prepareIndexIfNeededThenExecute(listener::onFailure, () -> { executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, - client.prepareUpdate(SECURITY_INDEX_NAME, INDEX_TYPE, + client.prepareUpdate(SECURITY_ALIAS_NAME, INDEX_TYPE, getIdForUser(USER_DOC_TYPE, username)) .setDoc(Requests.INDEX_CONTENT_TYPE, Fields.ENABLED.getPreferredName(), enabled) .setRefreshPolicy(refreshPolicy) @@ -450,7 +450,7 @@ private void setReservedUserEnabled(final String username, final boolean enabled boolean clearCache, final ActionListener listener) { securityIndex.prepareIndexIfNeededThenExecute(listener::onFailure, () -> { executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, - client.prepareUpdate(SECURITY_INDEX_NAME, INDEX_TYPE, + client.prepareUpdate(SECURITY_ALIAS_NAME, INDEX_TYPE, getIdForUser(RESERVED_USER_TYPE, username)) .setDoc(Requests.INDEX_CONTENT_TYPE, Fields.ENABLED.getPreferredName(), enabled) .setUpsert(XContentType.JSON, @@ -485,7 +485,7 @@ public void deleteUser(final DeleteUserRequest deleteUserRequest, final ActionLi listener.onFailure(frozenSecurityIndex.getUnavailableReason()); } else { securityIndex.checkIndexVersionThenExecute(listener::onFailure, () -> { - DeleteRequest request = client.prepareDelete(SECURITY_INDEX_NAME, + DeleteRequest request = client.prepareDelete(SECURITY_ALIAS_NAME, INDEX_TYPE, getIdForUser(USER_DOC_TYPE, deleteUserRequest.username())).request(); request.setRefreshPolicy(deleteUserRequest.getRefreshPolicy()); executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, request, @@ -532,7 +532,7 @@ void getReservedUserInfo(String username, ActionListener liste } else { securityIndex.checkIndexVersionThenExecute(listener::onFailure, () -> executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, - client.prepareGet(SECURITY_INDEX_NAME, INDEX_TYPE, + client.prepareGet(SECURITY_ALIAS_NAME, INDEX_TYPE, getIdForUser(RESERVED_USER_TYPE, username)).request(), new ActionListener() { @Override @@ -577,7 +577,7 @@ void getAllReservedUserInfo(ActionListener> listen } else { securityIndex.checkIndexVersionThenExecute(listener::onFailure, () -> executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, - client.prepareSearch(SECURITY_INDEX_NAME) + client.prepareSearch(SECURITY_ALIAS_NAME) .setQuery(QueryBuilders.termQuery(Fields.TYPE.getPreferredName(), RESERVED_USER_TYPE)) .setFetchSource(true).request(), new ActionListener() { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java index 3181c14fc272d..c0a7a243d9b19 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java @@ -60,13 +60,13 @@ import static org.elasticsearch.xpack.core.ClientHelper.SECURITY_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; import static org.elasticsearch.xpack.core.ClientHelper.stashWithOrigin; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_ALIAS_NAME; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.isIndexDeleted; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.isMoveFromRedToNonRed; /** * This store reads + writes {@link ExpressionRoleMapping role mappings} in an Elasticsearch - * {@link SecurityIndexManager#SECURITY_INDEX_NAME index}. + * {@link SecurityIndexManager#SECURITY_ALIAS_NAME index}. *
* The store is responsible for all read and write operations as well as * {@link #resolveRoles(UserData, ActionListener) resolving roles}. @@ -131,7 +131,7 @@ void loadMappings(ActionListener> listener) { final QueryBuilder query = QueryBuilders.termQuery(DOC_TYPE_FIELD, DOC_TYPE_ROLE_MAPPING); final Supplier supplier = client.threadPool().getThreadContext().newRestorableContext(false); try (ThreadContext.StoredContext ignore = stashWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN)) { - SearchRequest request = client.prepareSearch(SECURITY_INDEX_NAME) + SearchRequest request = client.prepareSearch(SECURITY_ALIAS_NAME) .setScroll(DEFAULT_KEEPALIVE_SETTING.get(settings)) .setTypes(SECURITY_GENERIC_TYPE) .setQuery(query) @@ -144,7 +144,7 @@ void loadMappings(ActionListener> listener) { listener.onResponse(mappings.stream().filter(Objects::nonNull).collect(Collectors.toList())), ex -> { logger.error(new ParameterizedMessage("failed to load role mappings from index [{}] skipping all mappings.", - SECURITY_INDEX_NAME), ex); + SECURITY_ALIAS_NAME), ex); listener.onResponse(Collections.emptyList()); })), doc -> buildMapping(getNameFromId(doc.getId()), doc.getSourceRef())); @@ -203,7 +203,7 @@ private void innerPutMapping(PutRoleMappingRequest request, ActionListener { executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, - client.prepareDelete(SECURITY_INDEX_NAME, SECURITY_GENERIC_TYPE, getIdForName(request.getName())) + client.prepareDelete(SECURITY_ALIAS_NAME, SECURITY_GENERIC_TYPE, getIdForName(request.getName())) .setRefreshPolicy(request.getRefreshPolicy()) .request(), new ActionListener() { @@ -287,7 +287,7 @@ private void getMappings(ActionListener> listener) { logger.info("The security index is not yet available - no role mappings can be loaded"); if (logger.isDebugEnabled()) { logger.debug("Security Index [{}] [exists: {}] [available: {}] [mapping up to date: {}]", - SECURITY_INDEX_NAME, + SECURITY_ALIAS_NAME, securityIndex.indexExists(), securityIndex.isAvailable(), securityIndex.isMappingUpToDate() diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java index 35a2f5340492d..e2dbb52ec30bf 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java @@ -320,7 +320,7 @@ && isSuperuser(authentication.getUser()) == false) { // only the XPackUser is allowed to work with this index, but we should allow indices monitoring actions through for debugging // purposes. These monitor requests also sometimes resolve indices concretely and then requests them logger.debug("user [{}] attempted to directly perform [{}] against the security index [{}]", - authentication.getUser().principal(), action, SecurityIndexManager.SECURITY_INDEX_NAME); + authentication.getUser().principal(), action, SecurityIndexManager.SECURITY_ALIAS_NAME); throw denial(auditId, authentication, action, request, permission.names()); } else { putTransientIfNonExisting(AuthorizationServiceField.INDICES_PERMISSIONS_KEY, indicesAccessControl); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStore.java index 63b3ba2c9bba7..b88ae28acc082 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStore.java @@ -61,7 +61,7 @@ import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; import static org.elasticsearch.xpack.core.ClientHelper.stashWithOrigin; import static org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor.DOC_TYPE_VALUE; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_ALIAS_NAME; /** * {@code NativePrivilegeStore} is a store that reads/writes {@link ApplicationPrivilegeDescriptor} objects, @@ -122,7 +122,7 @@ public void getPrivileges(Collection applications, Collection na } final Supplier supplier = client.threadPool().getThreadContext().newRestorableContext(false); try (ThreadContext.StoredContext ignore = stashWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN)) { - SearchRequest request = client.prepareSearch(SECURITY_INDEX_NAME) + SearchRequest request = client.prepareSearch(SECURITY_ALIAS_NAME) .setScroll(DEFAULT_KEEPALIVE_SETTING.get(settings)) .setQuery(query) .setSize(1000) @@ -151,7 +151,7 @@ void getPrivilege(String application, String name, ActionListener executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, - client.prepareGet(SECURITY_INDEX_NAME, "doc", toDocId(application, name)).request(), + client.prepareGet(SECURITY_ALIAS_NAME, "doc", toDocId(application, name)).request(), new ActionListener() { @Override public void onResponse(GetResponse response) { @@ -202,7 +202,7 @@ private void innerPutPrivilege(ApplicationPrivilegeDescriptor privilege, WriteRe final String name = privilege.getName(); final XContentBuilder xContentBuilder = privilege.toXContent(jsonBuilder(), true); ClientHelper.executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, - client.prepareIndex(SECURITY_INDEX_NAME, "doc", toDocId(privilege.getApplication(), name)) + client.prepareIndex(SECURITY_ALIAS_NAME, "doc", toDocId(privilege.getApplication(), name)) .setSource(xContentBuilder) .setRefreshPolicy(refreshPolicy) .request(), listener, client::index); @@ -233,7 +233,7 @@ public void deletePrivileges(String application, Collection names, Write }, listener::onFailure), names.size(), Collections.emptyList()); for (String name : names) { ClientHelper.executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, - client.prepareDelete(SECURITY_INDEX_NAME, "doc", toDocId(application, name)) + client.prepareDelete(SECURITY_ALIAS_NAME, "doc", toDocId(application, name)) .setRefreshPolicy(refreshPolicy) .request(), groupListener, client::delete); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java index a36f830ceacbc..fbb2c68f2e980 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java @@ -70,7 +70,7 @@ import static org.elasticsearch.xpack.core.ClientHelper.stashWithOrigin; import static org.elasticsearch.xpack.core.security.SecurityField.setting; import static org.elasticsearch.xpack.core.security.authz.RoleDescriptor.ROLE_TYPE; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_ALIAS_NAME; /** * NativeRolesStore is a {@code RolesStore} that, instead of reading from a @@ -122,7 +122,7 @@ public void getRoleDescriptors(Set names, final ActionListener supplier = client.threadPool().getThreadContext().newRestorableContext(false); try (ThreadContext.StoredContext ignore = stashWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN)) { - SearchRequest request = client.prepareSearch(SecurityIndexManager.SECURITY_INDEX_NAME) + SearchRequest request = client.prepareSearch(SecurityIndexManager.SECURITY_ALIAS_NAME) .setScroll(DEFAULT_KEEPALIVE_SETTING.get(settings)) .setQuery(query) .setSize(1000) @@ -140,7 +140,7 @@ public void getRoleDescriptors(Set names, final ActionListener { final String[] roleIds = names.stream().map(NativeRolesStore::getIdForRole).toArray(String[]::new); - MultiGetRequest multiGetRequest = client.prepareMultiGet().add(SECURITY_INDEX_NAME, ROLE_DOC_TYPE, roleIds).request(); + MultiGetRequest multiGetRequest = client.prepareMultiGet().add(SECURITY_ALIAS_NAME, ROLE_DOC_TYPE, roleIds).request(); executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, multiGetRequest, ActionListener.wrap(mGetResponse -> { final MultiGetItemResponse[] responses = mGetResponse.getResponses(); @@ -176,7 +176,7 @@ public void deleteRole(final DeleteRoleRequest deleteRoleRequest, final ActionLi listener.onFailure(frozenSecurityIndex.getUnavailableReason()); } else { securityIndex.checkIndexVersionThenExecute(listener::onFailure, () -> { - DeleteRequest request = client.prepareDelete(SecurityIndexManager.SECURITY_INDEX_NAME, + DeleteRequest request = client.prepareDelete(SecurityIndexManager.SECURITY_ALIAS_NAME, ROLE_DOC_TYPE, getIdForRole(deleteRoleRequest.name())).request(); request.setRefreshPolicy(deleteRoleRequest.getRefreshPolicy()); executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, request, @@ -217,7 +217,7 @@ void innerPutRole(final PutRoleRequest request, final RoleDescriptor role, final listener.onFailure(e); return; } - final IndexRequest indexRequest = client.prepareIndex(SECURITY_INDEX_NAME, ROLE_DOC_TYPE, getIdForRole(role.getName())) + final IndexRequest indexRequest = client.prepareIndex(SECURITY_ALIAS_NAME, ROLE_DOC_TYPE, getIdForRole(role.getName())) .setSource(xContentBuilder) .setRefreshPolicy(request.getRefreshPolicy()) .request(); @@ -251,10 +251,10 @@ public void usageStats(ActionListener> listener) { securityIndex.checkIndexVersionThenExecute(listener::onFailure, () -> executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, client.prepareMultiSearch() - .add(client.prepareSearch(SecurityIndexManager.SECURITY_INDEX_NAME) + .add(client.prepareSearch(SecurityIndexManager.SECURITY_ALIAS_NAME) .setQuery(QueryBuilders.termQuery(RoleDescriptor.Fields.TYPE.getPreferredName(), ROLE_TYPE)) .setSize(0)) - .add(client.prepareSearch(SecurityIndexManager.SECURITY_INDEX_NAME) + .add(client.prepareSearch(SecurityIndexManager.SECURITY_ALIAS_NAME) .setQuery(QueryBuilders.boolQuery() .must(QueryBuilders.termQuery(RoleDescriptor.Fields.TYPE.getPreferredName(), ROLE_TYPE)) .must(QueryBuilders.boolQuery() @@ -264,7 +264,7 @@ public void usageStats(ActionListener> listener) { .should(existsQuery("indices.fields")))) .setSize(0) .setTerminateAfter(1)) - .add(client.prepareSearch(SecurityIndexManager.SECURITY_INDEX_NAME) + .add(client.prepareSearch(SecurityIndexManager.SECURITY_ALIAS_NAME) .setQuery(QueryBuilders.boolQuery() .must(QueryBuilders.termQuery(RoleDescriptor.Fields.TYPE.getPreferredName(), ROLE_TYPE)) .filter(existsQuery("indices.query"))) @@ -332,7 +332,7 @@ public void onFailure(Exception e) { private void executeGetRoleRequest(String role, ActionListener listener) { securityIndex.checkIndexVersionThenExecute(listener::onFailure, () -> executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, - client.prepareGet(SECURITY_INDEX_NAME, + client.prepareGet(SECURITY_ALIAS_NAME, ROLE_DOC_TYPE, getIdForRole(role)).request(), listener, client::get)); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java index ca82d043d3bfa..7d70b8d0bd5e5 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java @@ -72,33 +72,43 @@ public class SecurityIndexManager implements ClusterStateListener { public static final String SECURITY_VERSION_STRING = "security-version"; public static final String TEMPLATE_VERSION_PATTERN = Pattern.quote("${security.template.version}"); public static final String SECURITY_TEMPLATE_NAME = "security-index-template"; - public static final String SECURITY_INDEX_NAME = ".security"; + public static final String SECURITY_ALIAS_NAME = ".security"; private static final Logger LOGGER = LogManager.getLogger(SecurityIndexManager.class); private final String indexName; + private final int internalIndexFormat; + private final String aliasName; + private final String templateName; private final Client client; private final List> stateChangeListeners = new CopyOnWriteArrayList<>(); private volatile State indexState; - public SecurityIndexManager(Client client, String indexName, ClusterService clusterService) { - this(client, indexName, new State(false, false, false, false, null, null)); + public static SecurityIndexManager buildSecurityIndexManager(Client client, ClusterService clusterService) { + return new SecurityIndexManager(client, INTERNAL_SECURITY_INDEX, INTERNAL_INDEX_FORMAT, SECURITY_ALIAS_NAME, SECURITY_TEMPLATE_NAME, clusterService); + } + + public SecurityIndexManager(Client client, String indexName, int internalIndexFormat, String aliasName, String templateName, ClusterService clusterService) { + this(client, indexName, internalIndexFormat, aliasName, templateName, new State(false, false, false, false, null, null)); clusterService.addListener(this); } - private SecurityIndexManager(Client client, String indexName, State indexState) { + private SecurityIndexManager(Client client, String indexName, int internalIndexFormat, String aliasName, String templateName, State indexState) { this.client = client; this.indexName = indexName; + this.internalIndexFormat = internalIndexFormat; + this.aliasName = aliasName; + this.templateName = templateName; this.indexState = indexState; } public SecurityIndexManager freeze() { - return new SecurityIndexManager(null, indexName, indexState); + return new SecurityIndexManager(null, indexName, internalIndexFormat, aliasName, templateName, indexState); } public static List indexNames() { - return Collections.unmodifiableList(Arrays.asList(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)); + return Collections.unmodifiableList(Arrays.asList(SECURITY_ALIAS_NAME, INTERNAL_SECURITY_INDEX)); } public boolean checkMappingVersion(Predicate requiredVersion) { @@ -130,13 +140,13 @@ public boolean isMappingUpToDate() { public ElasticsearchException getUnavailableReason() { final State localState = this.indexState; if (localState.indexAvailable) { - throw new IllegalStateException("caller must make sure to use a frozen state and check indexAvailable"); + throw new IllegalStateException("caller must make sure to use a frozen state and check indexAvailable [" + indexName + "]"); } if (localState.indexExists) { - return new UnavailableShardsException(null, "at least one primary shard for the security index is unavailable"); + return new UnavailableShardsException(null, "at least one primary shard for the security index [" + indexName + "] is unavailable"); } else { - return new IndexNotFoundException(SECURITY_INDEX_NAME); + return new IndexNotFoundException(indexName); } } @@ -161,7 +171,7 @@ public void clusterChanged(ClusterChangedEvent event) { final IndexMetaData indexMetaData = resolveConcreteIndex(indexName, event.state().metaData()); final boolean indexExists = indexMetaData != null; final boolean isIndexUpToDate = indexExists == false || - INDEX_FORMAT_SETTING.get(indexMetaData.getSettings()).intValue() == INTERNAL_INDEX_FORMAT; + INDEX_FORMAT_SETTING.get(indexMetaData.getSettings()).intValue() == internalIndexFormat; final boolean indexAvailable = checkIndexAvailable(event.state()); final boolean mappingIsUpToDate = indexExists == false || checkIndexMappingUpToDate(event.state()); final Version mappingVersion = oldestIndexMappingVersion(event.state()); @@ -293,8 +303,8 @@ public void prepareIndexIfNeededThenExecute(final Consumer consumer, "the upgrade API is run on the security index")); } else if (indexState.indexExists == false) { Tuple mappingAndSettings = loadMappingAndSettingsSourceFromTemplate(); - CreateIndexRequest request = new CreateIndexRequest(INTERNAL_SECURITY_INDEX) - .alias(new Alias(SECURITY_INDEX_NAME)) + CreateIndexRequest request = new CreateIndexRequest(indexName) + .alias(new Alias(aliasName)) .mapping("doc", mappingAndSettings.v1(), XContentType.JSON) .waitForActiveShards(ActiveShardCount.ALL) .settings(mappingAndSettings.v2()); @@ -322,7 +332,7 @@ public void onFailure(Exception e) { } }, client.admin().indices()::create); } else if (indexState.mappingUpToDate == false) { - PutMappingRequest request = new PutMappingRequest(INTERNAL_SECURITY_INDEX) + PutMappingRequest request = new PutMappingRequest(indexName) .source(loadMappingAndSettingsSourceFromTemplate().v1(), XContentType.JSON) .type("doc"); executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, request, @@ -339,9 +349,9 @@ public void onFailure(Exception e) { } private Tuple loadMappingAndSettingsSourceFromTemplate() { - final byte[] template = TemplateUtils.loadTemplate("/" + SECURITY_TEMPLATE_NAME + ".json", + final byte[] template = TemplateUtils.loadTemplate("/" + templateName + ".json", Version.CURRENT.toString(), SecurityIndexManager.TEMPLATE_VERSION_PATTERN).getBytes(StandardCharsets.UTF_8); - PutIndexTemplateRequest request = new PutIndexTemplateRequest(SECURITY_TEMPLATE_NAME).source(template, XContentType.JSON); + PutIndexTemplateRequest request = new PutIndexTemplateRequest(templateName).source(template, XContentType.JSON); return new Tuple<>(request.mappings().get("doc"), request.settings()); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClearRolesCacheTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClearRolesCacheTests.java index 79d1d19c50e8a..7e8f62f57eac3 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClearRolesCacheTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClearRolesCacheTests.java @@ -56,7 +56,7 @@ public void setupForTests() { logger.debug("--> created role [{}]", role); } - ensureGreen(SecurityIndexManager.SECURITY_INDEX_NAME); + ensureGreen(SecurityIndexManager.SECURITY_ALIAS_NAME); final Set rolesSet = new HashSet<>(Arrays.asList(roles)); // warm up the caches on every node diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java index 5d227de626484..348967c57d03b 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java @@ -67,7 +67,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoTimeout; import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_ALIAS_NAME; import static org.hamcrest.Matchers.is; import static org.hamcrest.core.IsCollectionContaining.hasItem; @@ -492,7 +492,7 @@ public void assertSecurityIndexActive(TestCluster testCluster) throws Exception assertTrue( "security index mapping not sufficient to read:\n" + Strings.toString(clusterState.toXContent(builder, ToXContent.EMPTY_PARAMS).endObject()), - SecurityIndexManager.checkIndexMappingVersionMatches(SECURITY_INDEX_NAME, clusterState, + SecurityIndexManager.checkIndexMappingVersionMatches(SECURITY_ALIAS_NAME, clusterState, Version.CURRENT.minimumIndexCompatibilityVersion()::onOrBefore)); Index securityIndex = resolveSecurityIndex(clusterState.metaData()); if (securityIndex != null) { @@ -510,7 +510,7 @@ protected void deleteSecurityIndex() { UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_SUPERUSER, SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING))); GetIndexRequest getIndexRequest = new GetIndexRequest(); - getIndexRequest.indices(SECURITY_INDEX_NAME); + getIndexRequest.indices(SECURITY_ALIAS_NAME); getIndexRequest.indicesOptions(IndicesOptions.lenientExpandOpen()); GetIndexResponse getIndexResponse = client.admin().indices().getIndex(getIndexRequest).actionGet(); if (getIndexResponse.getIndices().length > 0) { @@ -521,7 +521,7 @@ protected void deleteSecurityIndex() { } private static Index resolveSecurityIndex(MetaData metaData) { - final AliasOrIndex aliasOrIndex = metaData.getAliasAndIndexLookup().get(SECURITY_INDEX_NAME); + final AliasOrIndex aliasOrIndex = metaData.getAliasAndIndexLookup().get(SECURITY_ALIAS_NAME); if (aliasOrIndex != null) { return aliasOrIndex.getIndices().get(0).getIndex(); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java index 8674a5b295085..1f21711d146cc 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java @@ -68,7 +68,7 @@ import static org.elasticsearch.discovery.DiscoveryModule.ZEN2_DISCOVERY_TYPE; import static org.elasticsearch.discovery.DiscoveryModule.ZEN_DISCOVERY_TYPE; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.INTERNAL_INDEX_FORMAT; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_ALIAS_NAME; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; @@ -340,7 +340,7 @@ public void testIndexJoinValidator_Old_And_Rolling() throws Exception { BiConsumer joinValidator = security.getJoinValidator(); assertNotNull(joinValidator); DiscoveryNode node = new DiscoveryNode("foo", buildNewFakeTransportAddress(), Version.CURRENT); - IndexMetaData indexMetaData = IndexMetaData.builder(SECURITY_INDEX_NAME) + IndexMetaData indexMetaData = IndexMetaData.builder(SECURITY_ALIAS_NAME) .settings(settings(Version.V_6_1_0).put(INDEX_FORMAT_SETTING.getKey(), INTERNAL_INDEX_FORMAT - 1)) .numberOfShards(1).numberOfReplicas(0) .build(); @@ -361,7 +361,7 @@ public void testIndexJoinValidator_FullyCurrentCluster() throws Exception { assertNotNull(joinValidator); DiscoveryNode node = new DiscoveryNode("foo", buildNewFakeTransportAddress(), Version.CURRENT); int indexFormat = randomBoolean() ? INTERNAL_INDEX_FORMAT : INTERNAL_INDEX_FORMAT - 1; - IndexMetaData indexMetaData = IndexMetaData.builder(SECURITY_INDEX_NAME) + IndexMetaData indexMetaData = IndexMetaData.builder(SECURITY_ALIAS_NAME) .settings(settings(Version.V_6_1_0).put(INDEX_FORMAT_SETTING.getKey(), indexFormat)) .numberOfShards(1).numberOfReplicas(0) .build(); @@ -379,7 +379,7 @@ public void testIndexUpgradeValidatorWithUpToDateIndex() throws Exception { assertNotNull(joinValidator); Version version = randomBoolean() ? Version.CURRENT : Version.V_6_1_0; DiscoveryNode node = new DiscoveryNode("foo", buildNewFakeTransportAddress(), Version.CURRENT); - IndexMetaData indexMetaData = IndexMetaData.builder(SECURITY_INDEX_NAME) + IndexMetaData indexMetaData = IndexMetaData.builder(SECURITY_ALIAS_NAME) .settings(settings(version).put(INDEX_FORMAT_SETTING.getKey(), INTERNAL_INDEX_FORMAT)) .numberOfShards(1).numberOfReplicas(0) .build(); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java index ed965146eb670..cf448bf388489 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java @@ -149,7 +149,7 @@ public void testExpiredTokensDeletedAfterExpiration() throws Exception { assertThat(invalidateResponse.getResult().getErrors().size(), equalTo(0)); AtomicReference docId = new AtomicReference<>(); assertBusy(() -> { - SearchResponse searchResponse = client.prepareSearch(SecurityIndexManager.SECURITY_INDEX_NAME) + SearchResponse searchResponse = client.prepareSearch(SecurityIndexManager.SECURITY_ALIAS_NAME) .setSource(SearchSourceBuilder.searchSource() .query(QueryBuilders.termQuery("doc_type", "token"))) .setSize(1) @@ -162,7 +162,7 @@ public void testExpiredTokensDeletedAfterExpiration() throws Exception { // hack doc to modify the creation time to the day before Instant dayBefore = created.minus(1L, ChronoUnit.DAYS); assertTrue(Instant.now().isAfter(dayBefore)); - client.prepareUpdate(SecurityIndexManager.SECURITY_INDEX_NAME, "doc", docId.get()) + client.prepareUpdate(SecurityIndexManager.SECURITY_ALIAS_NAME, "doc", docId.get()) .setDoc("creation_time", dayBefore.toEpochMilli()) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .get(); @@ -180,8 +180,8 @@ public void testExpiredTokensDeletedAfterExpiration() throws Exception { assertEquals("token malformed", e.getMessage()); } } - client.admin().indices().prepareRefresh(SecurityIndexManager.SECURITY_INDEX_NAME).get(); - SearchResponse searchResponse = client.prepareSearch(SecurityIndexManager.SECURITY_INDEX_NAME) + client.admin().indices().prepareRefresh(SecurityIndexManager.SECURITY_ALIAS_NAME).get(); + SearchResponse searchResponse = client.prepareSearch(SecurityIndexManager.SECURITY_ALIAS_NAME) .setSource(SearchSourceBuilder.searchSource() .query(QueryBuilders.termQuery("doc_type", "token"))) .setTerminateAfter(1) diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java index 7f3e2cfce9854..f2d2f9408c793 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java @@ -84,7 +84,7 @@ public void testRetrieveUsers() throws Exception { addedUsers.add(uname); } logger.error("--> waiting for .security index"); - ensureGreen(SecurityIndexManager.SECURITY_INDEX_NAME); + ensureGreen(SecurityIndexManager.SECURITY_ALIAS_NAME); MockTerminal t = new MockTerminal(); String username = nodeClientUsername(); @@ -133,7 +133,7 @@ public void testRetrieveRoles() throws Exception { addedRoles.add(rname); } logger.error("--> waiting for .security index"); - ensureGreen(SecurityIndexManager.SECURITY_INDEX_NAME); + ensureGreen(SecurityIndexManager.SECURITY_ALIAS_NAME); MockTerminal t = new MockTerminal(); String username = nodeClientUsername(); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java index af337cdc718aa..86bb522b00d11 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java @@ -60,7 +60,7 @@ import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoTimeout; import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_ALIAS_NAME; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.INTERNAL_SECURITY_INDEX; import static org.hamcrest.Matchers.arrayContaining; import static org.hamcrest.Matchers.containsString; @@ -138,7 +138,7 @@ public void testAddAndGetUser() throws Exception { logger.error("--> creating user"); c.preparePutUser("joe", "s3kirt".toCharArray(), hasher, "role1", "user").get(); logger.error("--> waiting for .security index"); - ensureGreen(SECURITY_INDEX_NAME); + ensureGreen(SECURITY_ALIAS_NAME); logger.info("--> retrieving user"); GetUsersResponse resp = c.prepareGetUsers("joe").get(); assertTrue("user should exist", resp.hasUsers()); @@ -193,7 +193,7 @@ public void testAddAndGetRole() throws Exception { .metadata(metadata) .get(); logger.error("--> waiting for .security index"); - ensureGreen(SECURITY_INDEX_NAME); + ensureGreen(SECURITY_ALIAS_NAME); logger.info("--> retrieving role"); GetRolesResponse resp = c.prepareGetRoles().names("test_role").get(); assertTrue("role should exist", resp.hasRoles()); @@ -244,7 +244,7 @@ public void testAddUserAndRoleThenAuth() throws Exception { logger.error("--> creating user"); c.preparePutUser("joe", "s3krit".toCharArray(), hasher, "test_role").get(); logger.error("--> waiting for .security index"); - ensureGreen(SECURITY_INDEX_NAME); + ensureGreen(SECURITY_ALIAS_NAME); logger.info("--> retrieving user"); GetUsersResponse resp = c.prepareGetUsers("joe").get(); assertTrue("user should exist", resp.hasUsers()); @@ -265,7 +265,7 @@ public void testUpdatingUserAndAuthentication() throws Exception { logger.error("--> creating user"); c.preparePutUser("joe", "s3krit".toCharArray(), hasher, SecuritySettingsSource.TEST_ROLE).get(); logger.error("--> waiting for .security index"); - ensureGreen(SECURITY_INDEX_NAME); + ensureGreen(SECURITY_ALIAS_NAME); logger.info("--> retrieving user"); GetUsersResponse resp = c.prepareGetUsers("joe").get(); assertTrue("user should exist", resp.hasUsers()); @@ -301,7 +301,7 @@ public void testCreateDeleteAuthenticate() { c.preparePutUser("joe", "s3krit".toCharArray(), hasher, SecuritySettingsSource.TEST_ROLE).get(); logger.error("--> waiting for .security index"); - ensureGreen(SECURITY_INDEX_NAME); + ensureGreen(SECURITY_ALIAS_NAME); logger.info("--> retrieving user"); GetUsersResponse resp = c.prepareGetUsers("joe").get(); assertTrue("user should exist", resp.hasUsers()); @@ -339,7 +339,7 @@ public void testCreateAndUpdateRole() { logger.error("--> creating user"); c.preparePutUser("joe", "s3krit".toCharArray(), hasher, "test_role").get(); logger.error("--> waiting for .security index"); - ensureGreen(SECURITY_INDEX_NAME); + ensureGreen(SECURITY_ALIAS_NAME); if (authenticate) { final String token = basicAuthHeaderValue("joe", new SecureString("s3krit".toCharArray())); @@ -389,7 +389,7 @@ public void testAuthenticateWithDeletedRole() { .get(); c.preparePutUser("joe", "s3krit".toCharArray(), hasher, "test_role").get(); logger.error("--> waiting for .security index"); - ensureGreen(SECURITY_INDEX_NAME); + ensureGreen(SECURITY_ALIAS_NAME); final String token = basicAuthHeaderValue("joe", new SecureString("s3krit".toCharArray())); ClusterHealthResponse response = client().filterWithHeader(Collections.singletonMap("Authorization", token)).admin().cluster() @@ -520,7 +520,7 @@ public void testUsersAndRolesDoNotInterfereWithIndicesStats() throws Exception { .get(); } - IndicesStatsResponse response = client().admin().indices().prepareStats("foo", SECURITY_INDEX_NAME).get(); + IndicesStatsResponse response = client().admin().indices().prepareStats("foo", SECURITY_ALIAS_NAME).get(); assertThat(response.getFailedShards(), is(0)); assertThat(response.getIndices().size(), is(2)); assertThat(response.getIndices().get(INTERNAL_SECURITY_INDEX), notNullValue()); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java index 3d13119292b6c..175d26fc6a084 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java @@ -110,7 +110,7 @@ public void testBlankPasswordInIndexImpliesDefaultPassword() throws Exception { values.put(PASSWORD_FIELD, BLANK_PASSWORD); final GetResult result = new GetResult( - SecurityIndexManager.SECURITY_INDEX_NAME, + SecurityIndexManager.SECURITY_ALIAS_NAME, NativeUsersStore.INDEX_TYPE, NativeUsersStore.getIdForUser(NativeUsersStore.RESERVED_USER_TYPE, randomAlphaOfLength(12)), 0, 1, 1L, @@ -179,7 +179,7 @@ public void testVerifyNonExistentUser() throws Exception { nativeUsersStore.verifyPassword(username, password, future); final GetResult getResult = new GetResult( - SecurityIndexManager.SECURITY_INDEX_NAME, + SecurityIndexManager.SECURITY_ALIAS_NAME, NativeUsersStore.INDEX_TYPE, NativeUsersStore.getIdForUser(NativeUsersStore.USER_DOC_TYPE, username), UNASSIGNED_SEQ_NO, 0, 1L, @@ -221,7 +221,7 @@ private void respondToGetUserRequest(String username, SecureString password, Str values.put(User.Fields.TYPE.getPreferredName(), NativeUsersStore.USER_DOC_TYPE); final BytesReference source = BytesReference.bytes(jsonBuilder().map(values)); final GetResult getResult = new GetResult( - SecurityIndexManager.SECURITY_INDEX_NAME, + SecurityIndexManager.SECURITY_ALIAS_NAME, NativeUsersStore.INDEX_TYPE, NativeUsersStore.getIdForUser(NativeUsersStore.USER_DOC_TYPE, username), 0, 1, 1L, diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java index 2a9d832f0f012..344de42cf41e1 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java @@ -150,7 +150,7 @@ import static org.elasticsearch.test.SecurityTestsUtils.assertAuthenticationException; import static org.elasticsearch.test.SecurityTestsUtils.assertThrowsAuthorizationException; import static org.elasticsearch.test.SecurityTestsUtils.assertThrowsAuthorizationExceptionRunAs; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_ALIAS_NAME; import static org.hamcrest.Matchers.arrayContaining; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.endsWith; @@ -805,7 +805,7 @@ public void testNonXPackUserCannotExecuteOperationAgainstSecurityIndex() { ClusterState state = mock(ClusterState.class); when(clusterService.state()).thenReturn(state); when(state.metaData()).thenReturn(MetaData.builder() - .put(new IndexMetaData.Builder(SECURITY_INDEX_NAME) + .put(new IndexMetaData.Builder(SECURITY_ALIAS_NAME) .settings(Settings.builder().put("index.version.created", Version.CURRENT).build()) .numberOfShards(1).numberOfReplicas(0).build(), true) .build()); @@ -813,21 +813,21 @@ public void testNonXPackUserCannotExecuteOperationAgainstSecurityIndex() { List> requests = new ArrayList<>(); requests.add(new Tuple<>(BulkAction.NAME + "[s]", - new DeleteRequest(SECURITY_INDEX_NAME, "type", "id"))); + new DeleteRequest(SECURITY_ALIAS_NAME, "type", "id"))); requests.add(new Tuple<>(UpdateAction.NAME, - new UpdateRequest(SECURITY_INDEX_NAME, "type", "id"))); + new UpdateRequest(SECURITY_ALIAS_NAME, "type", "id"))); requests.add(new Tuple<>(BulkAction.NAME + "[s]", - new IndexRequest(SECURITY_INDEX_NAME, "type", "id"))); - requests.add(new Tuple<>(SearchAction.NAME, new SearchRequest(SECURITY_INDEX_NAME))); + new IndexRequest(SECURITY_ALIAS_NAME, "type", "id"))); + requests.add(new Tuple<>(SearchAction.NAME, new SearchRequest(SECURITY_ALIAS_NAME))); requests.add(new Tuple<>(TermVectorsAction.NAME, - new TermVectorsRequest(SECURITY_INDEX_NAME, "type", "id"))); - requests.add(new Tuple<>(GetAction.NAME, new GetRequest(SECURITY_INDEX_NAME, "type", "id"))); + new TermVectorsRequest(SECURITY_ALIAS_NAME, "type", "id"))); + requests.add(new Tuple<>(GetAction.NAME, new GetRequest(SECURITY_ALIAS_NAME, "type", "id"))); requests.add(new Tuple<>(TermVectorsAction.NAME, - new TermVectorsRequest(SECURITY_INDEX_NAME, "type", "id"))); + new TermVectorsRequest(SECURITY_ALIAS_NAME, "type", "id"))); requests.add(new Tuple<>(IndicesAliasesAction.NAME, new IndicesAliasesRequest() - .addAliasAction(AliasActions.add().alias("security_alias").index(SECURITY_INDEX_NAME)))); + .addAliasAction(AliasActions.add().alias("security_alias").index(SECURITY_ALIAS_NAME)))); requests.add( - new Tuple<>(UpdateSettingsAction.NAME, new UpdateSettingsRequest().indices(SECURITY_INDEX_NAME))); + new Tuple<>(UpdateSettingsAction.NAME, new UpdateSettingsRequest().indices(SECURITY_ALIAS_NAME))); for (Tuple requestTuple : requests) { String action = requestTuple.v1(); @@ -840,12 +840,12 @@ public void testNonXPackUserCannotExecuteOperationAgainstSecurityIndex() { } // we should allow waiting for the health of the index or any index if the user has this permission - ClusterHealthRequest request = new ClusterHealthRequest(SECURITY_INDEX_NAME); + ClusterHealthRequest request = new ClusterHealthRequest(SECURITY_ALIAS_NAME); authorize(authentication, ClusterHealthAction.NAME, request); verify(auditTrail).accessGranted(requestId, authentication, ClusterHealthAction.NAME, request, new String[]{role.getName()}); // multiple indices - request = new ClusterHealthRequest(SECURITY_INDEX_NAME, "foo", "bar"); + request = new ClusterHealthRequest(SECURITY_ALIAS_NAME, "foo", "bar"); authorize(authentication, ClusterHealthAction.NAME, request); verify(auditTrail).accessGranted(requestId, authentication, ClusterHealthAction.NAME, request, new String[]{role.getName()}); verifyNoMoreInteractions(auditTrail); @@ -864,21 +864,21 @@ public void testGrantedNonXPackUserCanExecuteMonitoringOperationsAgainstSecurity ClusterState state = mock(ClusterState.class); when(clusterService.state()).thenReturn(state); when(state.metaData()).thenReturn(MetaData.builder() - .put(new IndexMetaData.Builder(SECURITY_INDEX_NAME) + .put(new IndexMetaData.Builder(SECURITY_ALIAS_NAME) .settings(Settings.builder().put("index.version.created", Version.CURRENT).build()) .numberOfShards(1).numberOfReplicas(0).build(), true) .build()); final String requestId = AuditUtil.getOrGenerateRequestId(threadContext); List> requests = new ArrayList<>(); - requests.add(new Tuple<>(IndicesStatsAction.NAME, new IndicesStatsRequest().indices(SECURITY_INDEX_NAME))); - requests.add(new Tuple<>(RecoveryAction.NAME, new RecoveryRequest().indices(SECURITY_INDEX_NAME))); - requests.add(new Tuple<>(IndicesSegmentsAction.NAME, new IndicesSegmentsRequest().indices(SECURITY_INDEX_NAME))); - requests.add(new Tuple<>(GetSettingsAction.NAME, new GetSettingsRequest().indices(SECURITY_INDEX_NAME))); + requests.add(new Tuple<>(IndicesStatsAction.NAME, new IndicesStatsRequest().indices(SECURITY_ALIAS_NAME))); + requests.add(new Tuple<>(RecoveryAction.NAME, new RecoveryRequest().indices(SECURITY_ALIAS_NAME))); + requests.add(new Tuple<>(IndicesSegmentsAction.NAME, new IndicesSegmentsRequest().indices(SECURITY_ALIAS_NAME))); + requests.add(new Tuple<>(GetSettingsAction.NAME, new GetSettingsRequest().indices(SECURITY_ALIAS_NAME))); requests.add(new Tuple<>(IndicesShardStoresAction.NAME, - new IndicesShardStoresRequest().indices(SECURITY_INDEX_NAME))); + new IndicesShardStoresRequest().indices(SECURITY_ALIAS_NAME))); requests.add(new Tuple<>(UpgradeStatusAction.NAME, - new UpgradeStatusRequest().indices(SECURITY_INDEX_NAME))); + new UpgradeStatusRequest().indices(SECURITY_ALIAS_NAME))); for (final Tuple requestTuple : requests) { final String action = requestTuple.v1(); @@ -894,7 +894,7 @@ public void testSuperusersCanExecuteOperationAgainstSecurityIndex() { ClusterState state = mock(ClusterState.class); when(clusterService.state()).thenReturn(state); when(state.metaData()).thenReturn(MetaData.builder() - .put(new IndexMetaData.Builder(SECURITY_INDEX_NAME) + .put(new IndexMetaData.Builder(SECURITY_ALIAS_NAME) .settings(Settings.builder().put("index.version.created", Version.CURRENT).build()) .numberOfShards(1).numberOfReplicas(0).build(), true) .build()); @@ -902,26 +902,26 @@ public void testSuperusersCanExecuteOperationAgainstSecurityIndex() { List> requests = new ArrayList<>(); requests.add(new Tuple<>(DeleteAction.NAME, - new DeleteRequest(SECURITY_INDEX_NAME, "type", "id"))); + new DeleteRequest(SECURITY_ALIAS_NAME, "type", "id"))); requests.add(new Tuple<>(BulkAction.NAME + "[s]", - createBulkShardRequest(SECURITY_INDEX_NAME, DeleteRequest::new))); + createBulkShardRequest(SECURITY_ALIAS_NAME, DeleteRequest::new))); requests.add(new Tuple<>(UpdateAction.NAME, - new UpdateRequest(SECURITY_INDEX_NAME, "type", "id"))); + new UpdateRequest(SECURITY_ALIAS_NAME, "type", "id"))); requests.add(new Tuple<>(IndexAction.NAME, - new IndexRequest(SECURITY_INDEX_NAME, "type", "id"))); + new IndexRequest(SECURITY_ALIAS_NAME, "type", "id"))); requests.add(new Tuple<>(BulkAction.NAME + "[s]", - createBulkShardRequest(SECURITY_INDEX_NAME, IndexRequest::new))); - requests.add(new Tuple<>(SearchAction.NAME, new SearchRequest(SECURITY_INDEX_NAME))); + createBulkShardRequest(SECURITY_ALIAS_NAME, IndexRequest::new))); + requests.add(new Tuple<>(SearchAction.NAME, new SearchRequest(SECURITY_ALIAS_NAME))); requests.add(new Tuple<>(TermVectorsAction.NAME, - new TermVectorsRequest(SECURITY_INDEX_NAME, "type", "id"))); - requests.add(new Tuple<>(GetAction.NAME, new GetRequest(SECURITY_INDEX_NAME, "type", "id"))); + new TermVectorsRequest(SECURITY_ALIAS_NAME, "type", "id"))); + requests.add(new Tuple<>(GetAction.NAME, new GetRequest(SECURITY_ALIAS_NAME, "type", "id"))); requests.add(new Tuple<>(TermVectorsAction.NAME, - new TermVectorsRequest(SECURITY_INDEX_NAME, "type", "id"))); + new TermVectorsRequest(SECURITY_ALIAS_NAME, "type", "id"))); requests.add(new Tuple<>(IndicesAliasesAction.NAME, new IndicesAliasesRequest() - .addAliasAction(AliasActions.add().alias("security_alias").index(SECURITY_INDEX_NAME)))); - requests.add(new Tuple<>(ClusterHealthAction.NAME, new ClusterHealthRequest(SECURITY_INDEX_NAME))); + .addAliasAction(AliasActions.add().alias("security_alias").index(SECURITY_ALIAS_NAME)))); + requests.add(new Tuple<>(ClusterHealthAction.NAME, new ClusterHealthRequest(SECURITY_ALIAS_NAME))); requests.add(new Tuple<>(ClusterHealthAction.NAME, - new ClusterHealthRequest(SECURITY_INDEX_NAME, "foo", "bar"))); + new ClusterHealthRequest(SECURITY_ALIAS_NAME, "foo", "bar"))); for (final Tuple requestTuple : requests) { final String action = requestTuple.v1(); @@ -939,7 +939,7 @@ public void testSuperusersCanExecuteOperationAgainstSecurityIndexWithWildcard() ClusterState state = mock(ClusterState.class); when(clusterService.state()).thenReturn(state); when(state.metaData()).thenReturn(MetaData.builder() - .put(new IndexMetaData.Builder(SECURITY_INDEX_NAME) + .put(new IndexMetaData.Builder(SECURITY_ALIAS_NAME) .settings(Settings.builder().put("index.version.created", Version.CURRENT).build()) .numberOfShards(1).numberOfReplicas(0).build(), true) .build()); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizedIndicesTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizedIndicesTests.java index c48ac4568989b..1eeb3e7fcc1e5 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizedIndicesTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizedIndicesTests.java @@ -85,7 +85,7 @@ public void testSecurityIndicesAreRemovedFromRegularUser() { MetaData metaData = MetaData.builder() .put(new IndexMetaData.Builder("an-index").settings(indexSettings).numberOfShards(1).numberOfReplicas(0).build(), true) .put(new IndexMetaData.Builder("another-index").settings(indexSettings).numberOfShards(1).numberOfReplicas(0).build(), true) - .put(new IndexMetaData.Builder(SecurityIndexManager.SECURITY_INDEX_NAME).settings(indexSettings) + .put(new IndexMetaData.Builder(SecurityIndexManager.SECURITY_ALIAS_NAME).settings(indexSettings) .numberOfShards(1).numberOfReplicas(0).build(), true) .build(); @@ -101,12 +101,12 @@ public void testSecurityIndicesAreNotRemovedFromSuperUsers() { MetaData metaData = MetaData.builder() .put(new IndexMetaData.Builder("an-index").settings(indexSettings).numberOfShards(1).numberOfReplicas(0).build(), true) .put(new IndexMetaData.Builder("another-index").settings(indexSettings).numberOfShards(1).numberOfReplicas(0).build(), true) - .put(new IndexMetaData.Builder(SecurityIndexManager.SECURITY_INDEX_NAME).settings(indexSettings) + .put(new IndexMetaData.Builder(SecurityIndexManager.SECURITY_ALIAS_NAME).settings(indexSettings) .numberOfShards(1).numberOfReplicas(0).build(), true) .build(); AuthorizedIndices authorizedIndices = new AuthorizedIndices(user, role, SearchAction.NAME, metaData); List list = authorizedIndices.get(); - assertThat(list, containsInAnyOrder("an-index", "another-index", SecurityIndexManager.SECURITY_INDEX_NAME)); + assertThat(list, containsInAnyOrder("an-index", "another-index", SecurityIndexManager.SECURITY_ALIAS_NAME)); } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java index 83edb189e2935..a78177101f31e 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java @@ -81,7 +81,7 @@ import java.util.Map; import java.util.Set; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_ALIAS_NAME; import static org.hamcrest.Matchers.arrayContaining; import static org.hamcrest.Matchers.arrayContainingInAnyOrder; import static org.hamcrest.Matchers.contains; @@ -121,7 +121,7 @@ public void setup() { indexNameExpressionResolver = new IndexNameExpressionResolver(); final boolean withAlias = randomBoolean(); - final String securityIndexName = SECURITY_INDEX_NAME + (withAlias ? "-" + randomAlphaOfLength(5) : ""); + final String securityIndexName = SECURITY_ALIAS_NAME + (withAlias ? "-" + randomAlphaOfLength(5) : ""); MetaData metaData = MetaData.builder() .put(indexBuilder("foo").putAlias(AliasMetaData.builder("foofoobar")) .putAlias(AliasMetaData.builder("foounauthorized")).settings(settings)) @@ -1223,14 +1223,14 @@ public void testXPackSecurityUserHasAccessToSecurityIndex() { { final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(XPackSecurityUser.INSTANCE, SearchAction.NAME); List indices = resolveIndices(request, authorizedIndices).getLocal(); - assertThat(indices, hasItem(SecurityIndexManager.SECURITY_INDEX_NAME)); + assertThat(indices, hasItem(SecurityIndexManager.SECURITY_ALIAS_NAME)); } { IndicesAliasesRequest aliasesRequest = new IndicesAliasesRequest(); - aliasesRequest.addAliasAction(AliasActions.add().alias("security_alias").index(SECURITY_INDEX_NAME)); + aliasesRequest.addAliasAction(AliasActions.add().alias("security_alias").index(SECURITY_ALIAS_NAME)); final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(XPackSecurityUser.INSTANCE, IndicesAliasesAction.NAME); List indices = resolveIndices(aliasesRequest, authorizedIndices).getLocal(); - assertThat(indices, hasItem(SecurityIndexManager.SECURITY_INDEX_NAME)); + assertThat(indices, hasItem(SecurityIndexManager.SECURITY_ALIAS_NAME)); } } @@ -1238,7 +1238,7 @@ public void testXPackUserDoesNotHaveAccessToSecurityIndex() { SearchRequest request = new SearchRequest(); final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(XPackUser.INSTANCE, SearchAction.NAME); List indices = resolveIndices(request, authorizedIndices).getLocal(); - assertThat(indices, not(hasItem(SecurityIndexManager.SECURITY_INDEX_NAME))); + assertThat(indices, not(hasItem(SecurityIndexManager.SECURITY_ALIAS_NAME))); } public void testNonXPackUserAccessingSecurityIndex() { @@ -1250,7 +1250,7 @@ public void testNonXPackUserAccessingSecurityIndex() { SearchRequest request = new SearchRequest(); final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(allAccessUser, SearchAction.NAME); List indices = resolveIndices(request, authorizedIndices).getLocal(); - assertThat(indices, not(hasItem(SecurityIndexManager.SECURITY_INDEX_NAME))); + assertThat(indices, not(hasItem(SecurityIndexManager.SECURITY_ALIAS_NAME))); } { @@ -1258,7 +1258,7 @@ public void testNonXPackUserAccessingSecurityIndex() { aliasesRequest.addAliasAction(AliasActions.add().alias("security_alias1").index("*")); final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(allAccessUser, IndicesAliasesAction.NAME); List indices = resolveIndices(aliasesRequest, authorizedIndices).getLocal(); - assertThat(indices, not(hasItem(SecurityIndexManager.SECURITY_INDEX_NAME))); + assertThat(indices, not(hasItem(SecurityIndexManager.SECURITY_ALIAS_NAME))); } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java index e2acbb81560bc..51bc1ad0ef3a1 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java @@ -124,7 +124,7 @@ public void testGetSinglePrivilegeByName() throws Exception { assertThat(requests, iterableWithSize(1)); assertThat(requests.get(0), instanceOf(GetRequest.class)); GetRequest request = (GetRequest) requests.get(0); - assertThat(request.index(), equalTo(SecurityIndexManager.SECURITY_INDEX_NAME)); + assertThat(request.index(), equalTo(SecurityIndexManager.SECURITY_ALIAS_NAME)); assertThat(request.type(), equalTo("doc")); assertThat(request.id(), equalTo("application-privilege_myapp:admin")); @@ -142,7 +142,7 @@ public void testGetMissingPrivilege() throws Exception { assertThat(requests, iterableWithSize(1)); assertThat(requests.get(0), instanceOf(GetRequest.class)); GetRequest request = (GetRequest) requests.get(0); - assertThat(request.index(), equalTo(SecurityIndexManager.SECURITY_INDEX_NAME)); + assertThat(request.index(), equalTo(SecurityIndexManager.SECURITY_ALIAS_NAME)); assertThat(request.type(), equalTo("doc")); assertThat(request.id(), equalTo("application-privilege_myapp:admin")); @@ -165,7 +165,7 @@ public void testGetPrivilegesByApplicationName() throws Exception { assertThat(requests, iterableWithSize(1)); assertThat(requests.get(0), instanceOf(SearchRequest.class)); SearchRequest request = (SearchRequest) requests.get(0); - assertThat(request.indices(), arrayContaining(SecurityIndexManager.SECURITY_INDEX_NAME)); + assertThat(request.indices(), arrayContaining(SecurityIndexManager.SECURITY_ALIAS_NAME)); final String query = Strings.toString(request.source().query()); assertThat(query, containsString("{\"terms\":{\"application\":[\"myapp\",\"yourapp\"]")); @@ -192,7 +192,7 @@ public void testGetAllPrivileges() throws Exception { assertThat(requests, iterableWithSize(1)); assertThat(requests.get(0), instanceOf(SearchRequest.class)); SearchRequest request = (SearchRequest) requests.get(0); - assertThat(request.indices(), arrayContaining(SecurityIndexManager.SECURITY_INDEX_NAME)); + assertThat(request.indices(), arrayContaining(SecurityIndexManager.SECURITY_ALIAS_NAME)); final String query = Strings.toString(request.source().query()); assertThat(query, containsString("{\"term\":{\"type\":{\"value\":\"application-privilege\"")); @@ -228,7 +228,7 @@ public void testPutPrivileges() throws Exception { for (int i = 0; i < putPrivileges.size(); i++) { ApplicationPrivilegeDescriptor privilege = putPrivileges.get(i); IndexRequest request = indexRequests.get(i); - assertThat(request.indices(), arrayContaining(SecurityIndexManager.SECURITY_INDEX_NAME)); + assertThat(request.indices(), arrayContaining(SecurityIndexManager.SECURITY_ALIAS_NAME)); assertThat(request.type(), equalTo("doc")); assertThat(request.id(), equalTo( "application-privilege_" + privilege.getApplication() + ":" + privilege.getName() @@ -237,7 +237,7 @@ public void testPutPrivileges() throws Exception { assertThat(request.source(), equalTo(BytesReference.bytes(builder))); final boolean created = privilege.getName().equals("user") == false; indexListener.onResponse(new IndexResponse( - new ShardId(SecurityIndexManager.SECURITY_INDEX_NAME, uuid, i), + new ShardId(SecurityIndexManager.SECURITY_ALIAS_NAME, uuid, i), request.type(), request.id(), 1, 1, 1, created )); } @@ -273,12 +273,12 @@ public void testDeletePrivileges() throws Exception { for (int i = 0; i < privilegeNames.size(); i++) { String name = privilegeNames.get(i); DeleteRequest request = deletes.get(i); - assertThat(request.indices(), arrayContaining(SecurityIndexManager.SECURITY_INDEX_NAME)); + assertThat(request.indices(), arrayContaining(SecurityIndexManager.SECURITY_ALIAS_NAME)); assertThat(request.type(), equalTo("doc")); assertThat(request.id(), equalTo("application-privilege_app1:" + name)); final boolean found = name.equals("p2") == false; deleteListener.onResponse(new DeleteResponse( - new ShardId(SecurityIndexManager.SECURITY_INDEX_NAME, uuid, i), + new ShardId(SecurityIndexManager.SECURITY_ALIAS_NAME, uuid, i), request.type(), request.id(), 1, 1, 1, found )); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java index 0cd44e32e3c3d..1b86dc64145e2 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java @@ -56,7 +56,7 @@ import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_ALIAS_NAME; import static org.hamcrest.Matchers.arrayContaining; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; @@ -187,8 +187,7 @@ public void testPutOfRoleWithFlsDlsUnlicensed() throws IOException { final ClusterService clusterService = mock(ClusterService.class); final XPackLicenseState licenseState = mock(XPackLicenseState.class); final AtomicBoolean methodCalled = new AtomicBoolean(false); - final SecurityIndexManager securityIndex = - new SecurityIndexManager(client, SecurityIndexManager.SECURITY_INDEX_NAME, clusterService); + final SecurityIndexManager securityIndex = SecurityIndexManager.buildSecurityIndexManager(client, clusterService); final NativeRolesStore rolesStore = new NativeRolesStore(Settings.EMPTY, client, licenseState, securityIndex) { @Override void innerPutRole(final PutRoleRequest request, final RoleDescriptor role, final ActionListener listener) { @@ -248,7 +247,7 @@ void innerPutRole(final PutRoleRequest request, final RoleDescriptor role, final private ClusterState getClusterStateWithSecurityIndex() { final boolean withAlias = randomBoolean(); - final String securityIndexName = SECURITY_INDEX_NAME + (withAlias ? "-" + randomAlphaOfLength(5) : ""); + final String securityIndexName = SECURITY_ALIAS_NAME + (withAlias ? "-" + randomAlphaOfLength(5) : ""); Settings settings = Settings.builder() .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java index 0caeff4ddf64c..01d9672db864c 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java @@ -51,7 +51,7 @@ import org.hamcrest.Matchers; import org.junit.Before; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_ALIAS_NAME; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_TEMPLATE_NAME; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.TEMPLATE_VERSION_PATTERN; import static org.hamcrest.Matchers.equalTo; @@ -86,7 +86,7 @@ void doExecute(Action action, Request request, ActionListener SecurityIndexManager.checkIndexMappingVersionMatches(SECURITY_INDEX_NAME, clusterState, Version.CURRENT::equals)); - assertEquals("Cannot read security-version string in index " + SECURITY_INDEX_NAME, exception.getMessage()); + () -> SecurityIndexManager.checkIndexMappingVersionMatches(SECURITY_ALIAS_NAME, clusterState, Version.CURRENT::equals)); + assertEquals("Cannot read security-version string in index " + SECURITY_ALIAS_NAME, exception.getMessage()); } public void testIndexTemplateIsIdentifiedAsUpToDate() throws IOException { @@ -384,7 +384,7 @@ private ClusterState.Builder createClusterStateWithTemplate(String securityTempl private ClusterState.Builder createClusterStateWithMapping(String securityTemplateString) throws IOException { final ClusterState clusterState = createClusterStateWithIndex(securityTemplateString).build(); final String indexName = clusterState.metaData().getAliasAndIndexLookup() - .get(SECURITY_INDEX_NAME).getIndices().get(0).getIndex().getName(); + .get(SECURITY_ALIAS_NAME).getIndices().get(0).getIndex().getName(); return ClusterState.builder(clusterState).routingTable(SecurityTestUtils.buildIndexRoutingTable(indexName)); } @@ -418,7 +418,7 @@ private static IndexMetaData.Builder createIndexMetadata(String indexName, Strin private ClusterState.Builder createClusterStateWithIndex(String securityTemplate) throws IOException { final MetaData.Builder metaDataBuilder = new MetaData.Builder(); final boolean withAlias = randomBoolean(); - final String securityIndexName = SECURITY_INDEX_NAME + (withAlias ? "-" + randomAlphaOfLength(5) : ""); + final String securityIndexName = SECURITY_ALIAS_NAME + (withAlias ? "-" + randomAlphaOfLength(5) : ""); metaDataBuilder.put(createIndexMetadata(securityIndexName, securityTemplate)); ClusterState.Builder clusterStateBuilder = ClusterState.builder(state()); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/test/SecurityTestUtils.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/test/SecurityTestUtils.java index 12474b7a04d59..e7cfc02209a1e 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/test/SecurityTestUtils.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/test/SecurityTestUtils.java @@ -35,7 +35,7 @@ import static java.nio.file.StandardOpenOption.CREATE; import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING; import static java.nio.file.StandardOpenOption.WRITE; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_ALIAS_NAME; public class SecurityTestUtils { @@ -83,7 +83,7 @@ public static RoutingTable buildIndexRoutingTable(String indexName) { * Adds the index alias {@code .security} to the underlying concrete index. */ public static MetaData addAliasToMetaData(MetaData metaData, String indexName) { - AliasMetaData aliasMetaData = AliasMetaData.newAliasMetaDataBuilder(SECURITY_INDEX_NAME).build(); + AliasMetaData aliasMetaData = AliasMetaData.newAliasMetaDataBuilder(SECURITY_ALIAS_NAME).build(); MetaData.Builder metaDataBuilder = new MetaData.Builder(metaData); IndexMetaData indexMetaData = metaData.index(indexName); metaDataBuilder.put(IndexMetaData.builder(indexMetaData).putAlias(aliasMetaData)); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/XPackUserTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/XPackUserTests.java index e7b31d88eda19..182bc6587ddde 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/XPackUserTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/XPackUserTests.java @@ -31,7 +31,7 @@ public void testXPackUserCanAccessNonSecurityIndices() { public void testXPackUserCannotAccessSecurityIndex() { final String action = randomFrom(GetAction.NAME, SearchAction.NAME, IndexAction.NAME); final Predicate predicate = XPackUser.ROLE.indices().allowedIndicesMatcher(action); - assertThat(predicate.test(SecurityIndexManager.SECURITY_INDEX_NAME), Matchers.is(false)); + assertThat(predicate.test(SecurityIndexManager.SECURITY_ALIAS_NAME), Matchers.is(false)); assertThat(predicate.test(SecurityIndexManager.INTERNAL_SECURITY_INDEX), Matchers.is(false)); } From ddda03e1db6966ca74a50e00dc27a6866c924370 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Wed, 2 Jan 2019 22:55:28 +0200 Subject: [PATCH 05/19] Okay, let's go! --- .../xpack/security/Security.java | 11 +++--- .../xpack/security/authc/TokenService.java | 6 ++- .../support/SecurityIndexManager.java | 37 +++++++++++++------ .../xpack/security/SecurityTests.java | 10 ++--- .../support/SecurityIndexManagerTests.java | 6 +-- .../xpack/restart/FullClusterRestartIT.java | 2 +- 6 files changed, 45 insertions(+), 27 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index 062b804f55d65..48346f9278508 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -252,7 +252,7 @@ import static java.util.Collections.singletonList; import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_FORMAT_SETTING; import static org.elasticsearch.xpack.core.XPackSettings.HTTP_SSL_ENABLED; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.INTERNAL_INDEX_FORMAT; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.INTERNAL_SECURITY_INDEX_FORMAT; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_ALIAS_NAME; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_TEMPLATE_NAME; @@ -430,7 +430,8 @@ Collection createComponents(Client client, ThreadPool threadPool, Cluste securityIndex.set(SecurityIndexManager.buildSecurityIndexManager(client, clusterService)); - final TokenService tokenService = new TokenService(settings, Clock.systemUTC(), client, securityIndex.get(), clusterService); + final TokenService tokenService = new TokenService(settings, Clock.systemUTC(), client, securityIndex.get(), + SecurityIndexManager.buildSecurityTokensIndexManager(client, clusterService), clusterService); this.tokenService.set(tokenService); components.add(tokenService); @@ -1077,9 +1078,9 @@ static final class ValidateUpgradedSecurityIndex implements BiConsumer predicate) { - return loadIndexMappingVersions(indexName, clusterState, LOGGER).stream().allMatch(predicate); + return loadIndexMappingVersions(indexName, clusterState, logger).stream().allMatch(predicate); } private Version oldestIndexMappingVersion(ClusterState clusterState) { - final Set versions = loadIndexMappingVersions(indexName, clusterState, LOGGER); + final Set versions = loadIndexMappingVersions(indexName, clusterState, logger); return versions.stream().min(Version::compareTo).orElse(null); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java index 1f21711d146cc..c75319eca7129 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java @@ -67,7 +67,7 @@ import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_FORMAT_SETTING; import static org.elasticsearch.discovery.DiscoveryModule.ZEN2_DISCOVERY_TYPE; import static org.elasticsearch.discovery.DiscoveryModule.ZEN_DISCOVERY_TYPE; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.INTERNAL_INDEX_FORMAT; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.INTERNAL_SECURITY_INDEX_FORMAT; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_ALIAS_NAME; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; @@ -341,7 +341,7 @@ public void testIndexJoinValidator_Old_And_Rolling() throws Exception { assertNotNull(joinValidator); DiscoveryNode node = new DiscoveryNode("foo", buildNewFakeTransportAddress(), Version.CURRENT); IndexMetaData indexMetaData = IndexMetaData.builder(SECURITY_ALIAS_NAME) - .settings(settings(Version.V_6_1_0).put(INDEX_FORMAT_SETTING.getKey(), INTERNAL_INDEX_FORMAT - 1)) + .settings(settings(Version.V_6_1_0).put(INDEX_FORMAT_SETTING.getKey(), INTERNAL_SECURITY_INDEX_FORMAT - 1)) .numberOfShards(1).numberOfReplicas(0) .build(); DiscoveryNode existingOtherNode = new DiscoveryNode("bar", buildNewFakeTransportAddress(), Version.V_6_1_0); @@ -351,7 +351,7 @@ public void testIndexJoinValidator_Old_And_Rolling() throws Exception { .metaData(MetaData.builder().put(indexMetaData, true).build()).build(); IllegalStateException e = expectThrows(IllegalStateException.class, () -> joinValidator.accept(node, clusterState)); - assertThat(e.getMessage(), equalTo("Security index is not on the current version [6] - " + + assertThat(e.getMessage(), equalTo("Index [.security] is not on the current version [6]. " + "The Upgrade API must be run for 7.x nodes to join the cluster")); } @@ -360,7 +360,7 @@ public void testIndexJoinValidator_FullyCurrentCluster() throws Exception { BiConsumer joinValidator = security.getJoinValidator(); assertNotNull(joinValidator); DiscoveryNode node = new DiscoveryNode("foo", buildNewFakeTransportAddress(), Version.CURRENT); - int indexFormat = randomBoolean() ? INTERNAL_INDEX_FORMAT : INTERNAL_INDEX_FORMAT - 1; + int indexFormat = randomBoolean() ? INTERNAL_SECURITY_INDEX_FORMAT : INTERNAL_SECURITY_INDEX_FORMAT - 1; IndexMetaData indexMetaData = IndexMetaData.builder(SECURITY_ALIAS_NAME) .settings(settings(Version.V_6_1_0).put(INDEX_FORMAT_SETTING.getKey(), indexFormat)) .numberOfShards(1).numberOfReplicas(0) @@ -380,7 +380,7 @@ public void testIndexUpgradeValidatorWithUpToDateIndex() throws Exception { Version version = randomBoolean() ? Version.CURRENT : Version.V_6_1_0; DiscoveryNode node = new DiscoveryNode("foo", buildNewFakeTransportAddress(), Version.CURRENT); IndexMetaData indexMetaData = IndexMetaData.builder(SECURITY_ALIAS_NAME) - .settings(settings(version).put(INDEX_FORMAT_SETTING.getKey(), INTERNAL_INDEX_FORMAT)) + .settings(settings(version).put(INDEX_FORMAT_SETTING.getKey(), INTERNAL_SECURITY_INDEX_FORMAT)) .numberOfShards(1).numberOfReplicas(0) .build(); DiscoveryNode existingOtherNode = new DiscoveryNode("bar", buildNewFakeTransportAddress(), version); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java index 01d9672db864c..dd2d4d0e1aa33 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java @@ -208,7 +208,7 @@ public void testIndexOutOfDateListeners() throws Exception { // index doesn't exist and now exists with wrong format ClusterState.Builder clusterStateBuilder = createClusterState(INDEX_NAME, TEMPLATE_NAME, - SecurityIndexManager.INTERNAL_INDEX_FORMAT - 1); + SecurityIndexManager.INTERNAL_SECURITY_INDEX_FORMAT - 1); markShardsAvailable(clusterStateBuilder); manager.clusterChanged(event(clusterStateBuilder)); assertTrue(listenerCalled.get()); @@ -224,7 +224,7 @@ public void testIndexOutOfDateListeners() throws Exception { listenerCalled.set(false); // index doesn't exist and now exists with correct format - clusterStateBuilder = createClusterState(INDEX_NAME, TEMPLATE_NAME, SecurityIndexManager.INTERNAL_INDEX_FORMAT); + clusterStateBuilder = createClusterState(INDEX_NAME, TEMPLATE_NAME, SecurityIndexManager.INTERNAL_SECURITY_INDEX_FORMAT); markShardsAvailable(clusterStateBuilder); manager.clusterChanged(event(clusterStateBuilder)); assertTrue(listenerCalled.get()); @@ -245,7 +245,7 @@ private void assertIndexUpToDateButNotAvailable() { } public static ClusterState.Builder createClusterState(String indexName, String templateName) throws IOException { - return createClusterState(indexName, templateName, templateName, SecurityIndexManager.INTERNAL_INDEX_FORMAT); + return createClusterState(indexName, templateName, templateName, SecurityIndexManager.INTERNAL_SECURITY_INDEX_FORMAT); } public static ClusterState.Builder createClusterState(String indexName, String templateName, int format) throws IOException { diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java index beea8c9a21856..5b1da2a0f1430 100644 --- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java @@ -112,7 +112,7 @@ public void testSecurityNativeRealm() throws Exception { logger.info("settings map {}", settingsMap); if (settingsMap.containsKey("index")) { int format = Integer.parseInt(String.valueOf(((Map)settingsMap.get("index")).get("format"))); - needsUpgrade = format == SecurityIndexManager.INTERNAL_INDEX_FORMAT ? false : true; + needsUpgrade = format == SecurityIndexManager.INTERNAL_SECURITY_INDEX_FORMAT ? false : true; } else { needsUpgrade = true; } From 05de53b22a401bcb25d1c71c36bd73ac4286d5bd Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Thu, 3 Jan 2019 19:09:31 +0200 Subject: [PATCH 06/19] TokenService --- .../xpack/security/authc/TokenService.java | 83 +++++++++++-------- .../support/SecurityIndexManager.java | 27 ++++-- 2 files changed, 69 insertions(+), 41 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java index 8fd6399c3f5ad..976f33ff24a83 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java @@ -219,6 +219,14 @@ public static Boolean isTokenServiceEnabled(Settings settings) { return XPackSettings.TOKEN_SERVICE_ENABLED_SETTING.get(settings); } + private SecurityIndexManager indexManager() { + if (securityIndex.isIndexUpgraded()) { + return securityTokensIndex; + } else { + return securityIndex; + } + } + /** * Create a token based on the provided authentication and metadata. * The created token will be stored in the security index. @@ -265,13 +273,13 @@ public void createUserToken(Authentication authentication, Authentication origin .endObject(); builder.endObject(); final String documentId = getTokenDocumentId(userToken); - IndexRequest request = - client.prepareIndex(SecurityIndexManager.SECURITY_ALIAS_NAME, TYPE, documentId) + final SecurityIndexManager indexManager = indexManager(); + IndexRequest request = client.prepareIndex(indexManager.indexName(), TYPE, documentId) .setOpType(OpType.CREATE) .setSource(builder) .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL) .request(); - securityIndex.prepareIndexIfNeededThenExecute(ex -> listener.onFailure(traceLog("prepare security index", documentId, ex)), + indexManager.prepareIndexIfNeededThenExecute(ex -> listener.onFailure(traceLog("prepare security index", documentId, ex)), () -> executeAsyncWithOrigin(client, SECURITY_ORIGIN, IndexAction.INSTANCE, request, ActionListener.wrap(indexResponse -> listener.onResponse(new Tuple<>(userToken, refreshToken)), listener::onFailure)) @@ -355,14 +363,16 @@ void decodeToken(String token, ActionListener listener) throws IOExce final byte[] iv = in.readByteArray(); final Cipher cipher = getDecryptionCipher(iv, decodeKey, version, decodedSalt); decryptTokenId(in, cipher, version, ActionListener.wrap(tokenId -> { - if (securityIndex.isAvailable() == false) { - logger.warn("failed to get token [{}] since index is not available", tokenId); + final SecurityIndexManager indexManager = indexManager(); + if (indexManager.isAvailable() == false) { + logger.warn("failed to get token [{}] since index [{}] is not available", tokenId, + indexManager.indexName()); listener.onResponse(null); } else { - securityIndex.checkIndexVersionThenExecute( - ex -> listener.onFailure(traceLog("prepare security index", tokenId, ex)), + indexManager.checkIndexVersionThenExecute( + ex -> listener.onFailure(traceLog("prepare index [" + indexManager.indexName() + "]", tokenId, ex)), () -> { - final GetRequest getRequest = client.prepareGet(SecurityIndexManager.SECURITY_ALIAS_NAME, TYPE, + final GetRequest getRequest = client.prepareGet(indexManager.indexName(), TYPE, getTokenDocumentId(tokenId)).request(); Consumer onFailure = ex -> listener.onFailure(traceLog("decode token", tokenId, ex)); executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, getRequest, @@ -580,20 +590,21 @@ private void indexInvalidation(Collection tokenIds, ActionListener MAX_RETRY_ATTEMPTS) { - logger.warn("Failed to invalidate [{}] tokens after [{}] attempts", tokenIds.size(), - attemptCount.get()); + logger.warn("Failed to invalidate [{}] tokens after [{}] attempts", tokenIds.size(), attemptCount.get()); listener.onFailure(invalidGrantException("failed to invalidate tokens")); } else { BulkRequestBuilder bulkRequestBuilder = client.prepareBulk(); + SecurityIndexManager indexManager = indexManager(); for (String tokenId : tokenIds) { - UpdateRequest request = client.prepareUpdate(SecurityIndexManager.SECURITY_ALIAS_NAME, TYPE, getTokenDocumentId(tokenId)) + UpdateRequest request = client.prepareUpdate(indexManager.indexName(), TYPE, getTokenDocumentId(tokenId)) .setDoc(srcPrefix, Collections.singletonMap("invalidated", true)) .setFetchSource(srcPrefix, null) .request(); bulkRequestBuilder.add(request); } bulkRequestBuilder.setRefreshPolicy(RefreshPolicy.WAIT_UNTIL); - securityIndex.prepareIndexIfNeededThenExecute(ex -> listener.onFailure(traceLog("prepare security index", ex)), + indexManager.prepareIndexIfNeededThenExecute( + ex -> listener.onFailure(traceLog("prepare index [" + indexManager.indexName() + "]", ex)), () -> executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, bulkRequestBuilder.request(), ActionListener.wrap(bulkResponse -> { ArrayList retryTokenDocIds = new ArrayList<>(); @@ -669,24 +680,27 @@ private void findTokenFromRefreshToken(String refreshToken, ActionListener onFailure = ex -> listener.onFailure(traceLog("find by refresh token", refreshToken, ex)); - securityIndex.checkIndexVersionThenExecute(listener::onFailure, () -> + indexManager.checkIndexVersionThenExecute(listener::onFailure, () -> executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, request, ActionListener.wrap(searchResponse -> { if (searchResponse.isTimedOut()) { @@ -726,7 +740,8 @@ private void innerRefresh(String tokenDocId, Authentication userAuth, ActionList listener.onFailure(invalidGrantException("could not refresh the requested token")); } else { Consumer onFailure = ex -> listener.onFailure(traceLog("refresh token", tokenDocId, ex)); - GetRequest getRequest = client.prepareGet(SecurityIndexManager.SECURITY_ALIAS_NAME, TYPE, tokenDocId).request(); + final SecurityIndexManager indexManager = indexManager(); + GetRequest getRequest = client.prepareGet(indexManager.indexName(), TYPE, tokenDocId).request(); executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, getRequest, ActionListener.wrap(response -> { if (response.isExists()) { @@ -747,7 +762,7 @@ private void innerRefresh(String tokenDocId, Authentication userAuth, ActionList in.setVersion(authVersion); Authentication authentication = new Authentication(in); UpdateRequest updateRequest = - client.prepareUpdate(SecurityIndexManager.SECURITY_ALIAS_NAME, TYPE, tokenDocId) + client.prepareUpdate(indexManager.indexName(), TYPE, tokenDocId) .setVersion(response.getVersion()) .setDoc("refresh_token", Collections.singletonMap("refreshed", true)) .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL) @@ -757,10 +772,8 @@ private void innerRefresh(String tokenDocId, Authentication userAuth, ActionList updateResponse -> createUserToken(authentication, userAuth, listener, metadata, true), e -> { Throwable cause = ExceptionsHelper.unwrapCause(e); - if (cause instanceof VersionConflictEngineException || - isShardNotAvailableException(e)) { - innerRefresh(tokenDocId, userAuth, - listener, attemptCount); + if (cause instanceof VersionConflictEngineException || isShardNotAvailableException(e)) { + innerRefresh(tokenDocId, userAuth, listener, attemptCount); } else { onFailure.accept(e); } @@ -849,7 +862,8 @@ private Optional checkClient(Map public void findActiveTokensForRealm(String realmName, ActionListener>> listener, @Nullable Predicate> filter) { ensureEnabled(); - final SecurityIndexManager frozenSecurityIndex = securityIndex.freeze(); + final SecurityIndexManager indexManager = indexManager(); + final SecurityIndexManager frozenSecurityIndex = indexManager.freeze(); if (Strings.isNullOrEmpty(realmName)) { listener.onFailure(new IllegalArgumentException("Realm name is required")); } else if (frozenSecurityIndex.indexExists() == false) { @@ -872,7 +886,7 @@ public void findActiveTokensForRealm(String realmName, ActionListener>> listener) { ensureEnabled(); - - final SecurityIndexManager frozenSecurityIndex = securityIndex.freeze(); + final SecurityIndexManager indexManager = indexManager(); + final SecurityIndexManager frozenSecurityIndex = indexManager.freeze(); if (Strings.isNullOrEmpty(username)) { listener.onFailure(new IllegalArgumentException("username is required")); } else if (frozenSecurityIndex.indexExists() == false) { @@ -916,7 +930,7 @@ public void findActiveTokensForUser(String username, ActionListener ScrollHelper.fetchAllByEntity(client, request, listener, - (SearchHit hit) -> filterAndParseHit(hit, isOfUser(username)))); + () -> ScrollHelper.fetchAllByEntity(client, request, listener, (SearchHit hit) -> filterAndParseHit(hit, isOfUser(username)))); } } @@ -1020,14 +1033,14 @@ private void checkIfTokenIsValid(UserToken userToken, ActionListener if (currentTime.isAfter(userToken.getExpirationTime())) { listener.onFailure(traceLog("validate token", userToken.getId(), expiredTokenException())); } - if (securityIndex.indexExists() == false) { + final SecurityIndexManager indexManager = indexManager(); + if (indexManager.indexExists() == false) { // index doesn't exist so the token is considered invalid as we cannot verify its validity - logger.warn("failed to validate token [{}] since the security index doesn't exist", userToken.getId()); + logger.warn("failed to validate token [{}] since the index [{}] doesn't exist", userToken.getId(), indexManager.indexName()); listener.onResponse(null); } else { - securityIndex.checkIndexVersionThenExecute(listener::onFailure, () -> { - final GetRequest getRequest = client.prepareGet(SecurityIndexManager.SECURITY_ALIAS_NAME, TYPE, - getTokenDocumentId(userToken)).request(); + indexManager.checkIndexVersionThenExecute(listener::onFailure, () -> { + final GetRequest getRequest = client.prepareGet(indexManager.indexName(), TYPE, getTokenDocumentId(userToken)).request(); Consumer onFailure = ex -> listener.onFailure(traceLog("check token state", userToken.getId(), ex)); executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, getRequest, ActionListener.wrap(response -> { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java index 0b997a03e7f27..b83ff36ab5d69 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java @@ -104,7 +104,7 @@ public static SecurityIndexManager buildSecurityTokensIndexManager(Client client public SecurityIndexManager(Client client, String indexName, int internalIndexFormat, String aliasName, String templateName, ClusterService clusterService) { - this(client, indexName, internalIndexFormat, aliasName, templateName, new State(false, false, false, false, null, null)); + this(client, indexName, internalIndexFormat, aliasName, templateName, new State(false, false, false, false, false, null, null)); clusterService.addListener(this); } @@ -132,6 +132,10 @@ public boolean checkMappingVersion(Predicate requiredVersion) { return currentIndexState.mappingVersion == null || requiredVersion.test(currentIndexState.mappingVersion); } + public String indexName() { + return indexName; + } + public boolean indexExists() { return this.indexState.indexExists; } @@ -144,6 +148,10 @@ public boolean isIndexUpToDate() { return this.indexState.isIndexUpToDate; } + public boolean isIndexUpgraded() { + return this.indexState.isIndexUpgraded; + } + public boolean isAvailable() { return this.indexState.indexAvailable; } @@ -185,14 +193,17 @@ public void clusterChanged(ClusterChangedEvent event) { final State previousState = indexState; final IndexMetaData indexMetaData = resolveConcreteIndex(indexName, event.state().metaData()); final boolean indexExists = indexMetaData != null; - final boolean isIndexUpToDate = indexExists == false || - INDEX_FORMAT_SETTING.get(indexMetaData.getSettings()).intValue() == internalIndexFormat; + final boolean isIndexUpToDate = indexExists == false + || INDEX_FORMAT_SETTING.get(indexMetaData.getSettings()).intValue() >= internalIndexFormat; + final boolean isIndexUpgraded = indexExists == true + && INDEX_FORMAT_SETTING.get(indexMetaData.getSettings()).intValue() == internalIndexFormat + 1; final boolean indexAvailable = checkIndexAvailable(event.state()); final boolean mappingIsUpToDate = indexExists == false || checkIndexMappingUpToDate(event.state()); final Version mappingVersion = oldestIndexMappingVersion(event.state()); final ClusterHealthStatus indexStatus = indexMetaData == null ? null : new ClusterIndexHealth(indexMetaData, event.state().getRoutingTable().index(indexMetaData.getIndex())).getStatus(); - final State newState = new State(indexExists, isIndexUpToDate, indexAvailable, mappingIsUpToDate, mappingVersion, indexStatus); + final State newState = new State(indexExists, isIndexUpToDate, isIndexUpgraded, indexAvailable, mappingIsUpToDate, mappingVersion, + indexStatus); this.indexState = newState; if (newState.equals(previousState) == false) { @@ -391,15 +402,17 @@ public static boolean isIndexDeleted(State previousState, State currentState) { public static class State { public final boolean indexExists; public final boolean isIndexUpToDate; + public final boolean isIndexUpgraded; public final boolean indexAvailable; public final boolean mappingUpToDate; public final Version mappingVersion; public final ClusterHealthStatus indexStatus; - public State(boolean indexExists, boolean isIndexUpToDate, boolean indexAvailable, + public State(boolean indexExists, boolean isIndexUpToDate, boolean isIndexUpgraded, boolean indexAvailable, boolean mappingUpToDate, Version mappingVersion, ClusterHealthStatus indexStatus) { this.indexExists = indexExists; this.isIndexUpToDate = isIndexUpToDate; + this.isIndexUpgraded = isIndexUpgraded; this.indexAvailable = indexAvailable; this.mappingUpToDate = mappingUpToDate; this.mappingVersion = mappingVersion; @@ -413,6 +426,7 @@ public boolean equals(Object o) { State state = (State) o; return indexExists == state.indexExists && isIndexUpToDate == state.isIndexUpToDate && + isIndexUpgraded == state.isIndexUpgraded && indexAvailable == state.indexAvailable && mappingUpToDate == state.mappingUpToDate && Objects.equals(mappingVersion, state.mappingVersion) && @@ -421,7 +435,8 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(indexExists, isIndexUpToDate, indexAvailable, mappingUpToDate, mappingVersion, indexStatus); + return Objects.hash(indexExists, isIndexUpToDate, isIndexUpgraded, indexAvailable, mappingUpToDate, mappingVersion, + indexStatus); } } } From a433eb94534e8783a16c4f78de31264a925ea6f6 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Fri, 4 Jan 2019 15:31:00 +0200 Subject: [PATCH 07/19] Use alias to resolve the index in the SecurityManager --- .../upgrade/IndexUpgradeCheckVersion.java | 13 -- .../xpack/security/authc/TokenService.java | 41 +++--- .../authc/esnative/NativeUsersStore.java | 12 +- .../authc/esnative/ReservedRealm.java | 2 +- .../mapper/NativeRoleMappingStore.java | 10 +- .../authz/store/CompositeRolesStore.java | 2 +- .../authz/store/NativePrivilegeStore.java | 4 +- .../authz/store/NativeRolesStore.java | 6 +- .../support/SecurityIndexManager.java | 138 +++++++++--------- ...sportSamlInvalidateSessionActionTests.java | 2 +- .../authc/AuthenticationServiceTests.java | 4 +- .../security/authc/TokenServiceTests.java | 10 +- .../authc/esnative/NativeUsersStoreTests.java | 4 +- .../authc/esnative/ReservedRealmTests.java | 24 +-- .../store/NativePrivilegeStoreTests.java | 2 +- .../support/SecurityIndexManagerTests.java | 18 +-- .../xpack/upgrade/InternalIndexReindexer.java | 82 +++++++---- .../elasticsearch/xpack/upgrade/Upgrade.java | 2 +- 18 files changed, 189 insertions(+), 187 deletions(-) delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/upgrade/IndexUpgradeCheckVersion.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/upgrade/IndexUpgradeCheckVersion.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/upgrade/IndexUpgradeCheckVersion.java deleted file mode 100644 index e09f73a688e57..0000000000000 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/upgrade/IndexUpgradeCheckVersion.java +++ /dev/null @@ -1,13 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.core.upgrade; - -public final class IndexUpgradeCheckVersion { - public static final int UPRADE_VERSION = 6; - - private IndexUpgradeCheckVersion() {} - -} diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java index 976f33ff24a83..8330e0604834b 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java @@ -220,7 +220,7 @@ public static Boolean isTokenServiceEnabled(Settings settings) { } private SecurityIndexManager indexManager() { - if (securityIndex.isIndexUpgraded()) { + if (securityTokensIndex.exists()) { return securityTokensIndex; } else { return securityIndex; @@ -274,7 +274,7 @@ public void createUserToken(Authentication authentication, Authentication origin builder.endObject(); final String documentId = getTokenDocumentId(userToken); final SecurityIndexManager indexManager = indexManager(); - IndexRequest request = client.prepareIndex(indexManager.indexName(), TYPE, documentId) + IndexRequest request = client.prepareIndex(indexManager.aliasName(), TYPE, documentId) .setOpType(OpType.CREATE) .setSource(builder) .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL) @@ -365,14 +365,13 @@ void decodeToken(String token, ActionListener listener) throws IOExce decryptTokenId(in, cipher, version, ActionListener.wrap(tokenId -> { final SecurityIndexManager indexManager = indexManager(); if (indexManager.isAvailable() == false) { - logger.warn("failed to get token [{}] since index [{}] is not available", tokenId, - indexManager.indexName()); + logger.warn("Failed to get token [{}]. Alias [{}] is not available.", tokenId, indexManager.aliasName()); listener.onResponse(null); } else { indexManager.checkIndexVersionThenExecute( - ex -> listener.onFailure(traceLog("prepare index [" + indexManager.indexName() + "]", tokenId, ex)), + ex -> listener.onFailure(traceLog("prepare alias [" + indexManager.aliasName() + "]", tokenId, ex)), () -> { - final GetRequest getRequest = client.prepareGet(indexManager.indexName(), TYPE, + final GetRequest getRequest = client.prepareGet(indexManager.aliasName(), TYPE, getTokenDocumentId(tokenId)).request(); Consumer onFailure = ex -> listener.onFailure(traceLog("decode token", tokenId, ex)); executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, getRequest, @@ -596,7 +595,7 @@ private void indexInvalidation(Collection tokenIds, ActionListener tokenIds, ActionListener listener.onFailure(traceLog("prepare index [" + indexManager.indexName() + "]", ex)), + ex -> listener.onFailure(traceLog("prepare alias [" + indexManager.aliasName() + "]", ex)), () -> executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, bulkRequestBuilder.request(), ActionListener.wrap(bulkResponse -> { ArrayList retryTokenDocIds = new ArrayList<>(); @@ -681,7 +680,7 @@ private void findTokenFromRefreshToken(String refreshToken, ActionListener onFailure = ex -> listener.onFailure(traceLog("refresh token", tokenDocId, ex)); final SecurityIndexManager indexManager = indexManager(); - GetRequest getRequest = client.prepareGet(indexManager.indexName(), TYPE, tokenDocId).request(); + GetRequest getRequest = client.prepareGet(indexManager.aliasName(), TYPE, tokenDocId).request(); executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, getRequest, ActionListener.wrap(response -> { if (response.isExists()) { @@ -762,7 +761,7 @@ private void innerRefresh(String tokenDocId, Authentication userAuth, ActionList in.setVersion(authVersion); Authentication authentication = new Authentication(in); UpdateRequest updateRequest = - client.prepareUpdate(indexManager.indexName(), TYPE, tokenDocId) + client.prepareUpdate(indexManager.aliasName(), TYPE, tokenDocId) .setVersion(response.getVersion()) .setDoc("refresh_token", Collections.singletonMap("refreshed", true)) .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL) @@ -866,7 +865,7 @@ public void findActiveTokensForRealm(String realmName, ActionListener listener.onFailure(traceLog("validate token", userToken.getId(), expiredTokenException())); } final SecurityIndexManager indexManager = indexManager(); - if (indexManager.indexExists() == false) { + if (indexManager.exists() == false) { // index doesn't exist so the token is considered invalid as we cannot verify its validity - logger.warn("failed to validate token [{}] since the index [{}] doesn't exist", userToken.getId(), indexManager.indexName()); + logger.warn("failed to validate token [{}] since the alias [{}] doesn't exist", userToken.getId(), indexManager.aliasName()); listener.onResponse(null); } else { indexManager.checkIndexVersionThenExecute(listener::onFailure, () -> { - final GetRequest getRequest = client.prepareGet(indexManager.indexName(), TYPE, getTokenDocumentId(userToken)).request(); + final GetRequest getRequest = client.prepareGet(indexManager.aliasName(), TYPE, getTokenDocumentId(userToken)).request(); Consumer onFailure = ex -> listener.onFailure(traceLog("check token state", userToken.getId(), ex)); executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, getRequest, ActionListener.wrap(response -> { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java index 6d5b0a0c058bc..2552c2a3364bc 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java @@ -127,7 +127,7 @@ public void getUsers(String[] userNames, final ActionListener> }; final SecurityIndexManager frozenSecurityIndex = this.securityIndex.freeze(); - if (frozenSecurityIndex.indexExists() == false) { + if (frozenSecurityIndex.exists() == false) { listener.onResponse(Collections.emptyList()); } else if (frozenSecurityIndex.isAvailable() == false) { listener.onFailure(frozenSecurityIndex.getUnavailableReason()); @@ -165,7 +165,7 @@ public void getUsers(String[] userNames, final ActionListener> void getUserCount(final ActionListener listener) { final SecurityIndexManager frozenSecurityIndex = this.securityIndex.freeze(); - if (frozenSecurityIndex.indexExists() == false) { + if (frozenSecurityIndex.exists() == false) { listener.onResponse(0L); } else if (frozenSecurityIndex.isAvailable() == false) { listener.onFailure(frozenSecurityIndex.getUnavailableReason()); @@ -196,7 +196,7 @@ public void onFailure(Exception e) { private void getUserAndPassword(final String user, final ActionListener listener) { final SecurityIndexManager frozenSecurityIndex = securityIndex.freeze(); if (frozenSecurityIndex.isAvailable() == false) { - if (frozenSecurityIndex.indexExists()) { + if (frozenSecurityIndex.exists()) { logger.trace("could not retrieve user [{}] because security index does not exist", user); } else { logger.error("security index is unavailable. short circuiting retrieval of user [{}]", user); @@ -479,7 +479,7 @@ public void onFailure(Exception e) { public void deleteUser(final DeleteUserRequest deleteUserRequest, final ActionListener listener) { final SecurityIndexManager frozenSecurityIndex = securityIndex.freeze(); - if (frozenSecurityIndex.indexExists() == false) { + if (frozenSecurityIndex.exists() == false) { listener.onResponse(false); } else if (frozenSecurityIndex.isAvailable() == false) { listener.onFailure(frozenSecurityIndex.getUnavailableReason()); @@ -525,7 +525,7 @@ void verifyPassword(String username, final SecureString password, ActionListener void getReservedUserInfo(String username, ActionListener listener) { final SecurityIndexManager frozenSecurityIndex = securityIndex.freeze(); - if (frozenSecurityIndex.indexExists() == false) { + if (frozenSecurityIndex.exists() == false) { listener.onResponse(null); } else if (frozenSecurityIndex.isAvailable() == false) { listener.onFailure(frozenSecurityIndex.getUnavailableReason()); @@ -570,7 +570,7 @@ public void onFailure(Exception e) { void getAllReservedUserInfo(ActionListener> listener) { final SecurityIndexManager frozenSecurityIndex = securityIndex.freeze(); - if (frozenSecurityIndex.indexExists() == false) { + if (frozenSecurityIndex.exists() == false) { listener.onResponse(Collections.emptyMap()); } else if (frozenSecurityIndex.isAvailable() == false) { listener.onFailure(frozenSecurityIndex.getUnavailableReason()); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealm.java index 85e25925f4547..c17fc65b1dcc9 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealm.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealm.java @@ -206,7 +206,7 @@ private void getUserInfo(final String username, ActionListener if (userIsDefinedForCurrentSecurityMapping(username) == false) { logger.debug("Marking user [{}] as disabled because the security mapping is not at the required version", username); listener.onResponse(disabledDefaultUserInfo.deepClone()); - } else if (securityIndex.indexExists() == false) { + } else if (securityIndex.exists() == false) { listener.onResponse(getDefaultUserInfo(username)); } else { nativeUsersStore.getReservedUserInfo(username, ActionListener.wrap((userInfo) -> { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java index c0a7a243d9b19..4248e81278710 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java @@ -122,7 +122,7 @@ private String getIdForName(String name) { * package private for unit testing */ void loadMappings(ActionListener> listener) { - if (securityIndex.isIndexUpToDate() == false) { + if (securityIndex.isUpToDate() == false) { listener.onFailure(new IllegalStateException( "Security index is not on the current version - the native realm will not be operational until " + "the upgrade API is run on the security index")); @@ -178,7 +178,7 @@ public void deleteRoleMapping(DeleteRoleMappingRequest request, ActionListener void modifyMapping(String name, CheckedBiConsumer, Exception> inner, Request request, ActionListener listener) { - if (securityIndex.isIndexUpToDate() == false) { + if (securityIndex.isUpToDate() == false) { listener.onFailure(new IllegalStateException( "Security index is not on the current version - the native realm will not be operational until " + "the upgrade API is run on the security index")); @@ -225,7 +225,7 @@ public void onFailure(Exception e) { private void innerDeleteMapping(DeleteRoleMappingRequest request, ActionListener listener) { final SecurityIndexManager frozenSecurityIndex = securityIndex.freeze(); - if (frozenSecurityIndex.indexExists() == false) { + if (frozenSecurityIndex.exists() == false) { listener.onResponse(false); } else if (securityIndex.isAvailable() == false) { listener.onFailure(frozenSecurityIndex.getUnavailableReason()); @@ -288,7 +288,7 @@ private void getMappings(ActionListener> listener) { if (logger.isDebugEnabled()) { logger.debug("Security Index [{}] [exists: {}] [available: {}] [mapping up to date: {}]", SECURITY_ALIAS_NAME, - securityIndex.indexExists(), + securityIndex.exists(), securityIndex.isAvailable(), securityIndex.isMappingUpToDate() ); @@ -323,7 +323,7 @@ private void reportStats(ActionListener> listener, List> listener) { public void onSecurityIndexStateChange(SecurityIndexManager.State previousState, SecurityIndexManager.State currentState) { if (isMoveFromRedToNonRed(previousState, currentState) || isIndexDeleted(previousState, currentState) || - previousState.isIndexUpToDate != currentState.isIndexUpToDate) { + previousState.isUpToDate != currentState.isUpToDate) { invalidateAll(); } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStore.java index b88ae28acc082..ba664d70b56a2 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStore.java @@ -92,7 +92,7 @@ public NativePrivilegeStore(Settings settings, Client client, SecurityIndexManag public void getPrivileges(Collection applications, Collection names, ActionListener> listener) { final SecurityIndexManager frozenSecurityIndex = securityIndexManager.freeze(); - if (frozenSecurityIndex.indexExists() == false) { + if (frozenSecurityIndex.exists() == false) { listener.onResponse(Collections.emptyList()); } else if (frozenSecurityIndex.isAvailable() == false) { listener.onFailure(frozenSecurityIndex.getUnavailableReason()); @@ -216,7 +216,7 @@ private void innerPutPrivilege(ApplicationPrivilegeDescriptor privilege, WriteRe public void deletePrivileges(String application, Collection names, WriteRequest.RefreshPolicy refreshPolicy, ActionListener>> listener) { final SecurityIndexManager frozenSecurityIndex = securityIndexManager.freeze(); - if (frozenSecurityIndex.indexExists() == false) { + if (frozenSecurityIndex.exists() == false) { listener.onResponse(Collections.emptyMap()); } else if (frozenSecurityIndex.isAvailable() == false) { listener.onFailure(frozenSecurityIndex.getUnavailableReason()); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java index fbb2c68f2e980..7d338b8ebe93c 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java @@ -114,7 +114,7 @@ public void accept(Set names, ActionListener listen * Retrieve a list of roles, if rolesToGet is null or empty, fetch all roles */ public void getRoleDescriptors(Set names, final ActionListener listener) { - if (securityIndex.indexExists() == false) { + if (securityIndex.exists() == false) { // TODO remove this short circuiting and fix tests that fail without this! listener.onResponse(RoleRetrievalResult.success(Collections.emptySet())); } else if (names == null || names.isEmpty()) { @@ -170,7 +170,7 @@ public void getRoleDescriptors(Set names, final ActionListener listener) { final SecurityIndexManager frozenSecurityIndex = securityIndex.freeze(); - if (frozenSecurityIndex.indexExists() == false) { + if (frozenSecurityIndex.exists() == false) { listener.onResponse(false); } else if (frozenSecurityIndex.isAvailable() == false) { listener.onFailure(frozenSecurityIndex.getUnavailableReason()); @@ -308,7 +308,7 @@ public String toString() { } private void getRoleDescriptor(final String roleId, ActionListener resultListener) { - if (securityIndex.indexExists() == false) { + if (securityIndex.exists() == false) { // TODO remove this short circuiting and fix tests that fail without this! resultListener.onResponse(RoleRetrievalResult.success(Collections.emptySet())); } else { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java index b83ff36ab5d69..bc25b44aef1e4 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java @@ -41,7 +41,6 @@ import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.xpack.core.template.TemplateUtils; -import org.elasticsearch.xpack.core.upgrade.IndexUpgradeCheckVersion; import java.nio.charset.StandardCharsets; import java.util.Arrays; @@ -69,12 +68,12 @@ public class SecurityIndexManager implements ClusterStateListener { public static final String SECURITY_ALIAS_NAME = ".security"; public static final int INTERNAL_SECURITY_INDEX_FORMAT = 6; - public static final String INTERNAL_SECURITY_INDEX = ".security-" + IndexUpgradeCheckVersion.UPRADE_VERSION; + public static final String INTERNAL_SECURITY_INDEX = ".security-" + INTERNAL_SECURITY_INDEX_FORMAT; public static final String SECURITY_TEMPLATE_NAME = "security-index-template"; public static final String SECURITY_TOKENS_ALIAS_NAME = ".security-tokens"; - public static final String INTERNAL_SECURITY_TOKENS_INDEX = ".security-tokens-" + IndexUpgradeCheckVersion.UPRADE_VERSION; public static final int INTERNAL_SECURITY_TOKENS_INDEX_FORMAT = 7; + public static final String INTERNAL_SECURITY_TOKENS_INDEX = ".security-tokens-" + INTERNAL_SECURITY_TOKENS_INDEX_FORMAT; public static final String SECURITY_TOKENS_TEMPLATE_NAME = "security-tokens-index-template"; public static final String SECURITY_VERSION_STRING = "security-version"; @@ -83,7 +82,7 @@ public class SecurityIndexManager implements ClusterStateListener { private static final Logger logger = LogManager.getLogger(); private final String indexName; - private final int internalIndexFormat; + private final int expectedIndexFormat; private final String aliasName; private final String templateName; private final Client client; @@ -102,24 +101,24 @@ public static SecurityIndexManager buildSecurityTokensIndexManager(Client client SECURITY_TOKENS_ALIAS_NAME, SECURITY_TOKENS_TEMPLATE_NAME, clusterService); } - public SecurityIndexManager(Client client, String indexName, int internalIndexFormat, String aliasName, String templateName, + public SecurityIndexManager(Client client, String indexName, int expectedIndexFormat, String aliasName, String templateName, ClusterService clusterService) { - this(client, indexName, internalIndexFormat, aliasName, templateName, new State(false, false, false, false, false, null, null)); + this(client, indexName, expectedIndexFormat, aliasName, templateName, new State(false, false, false, false, null, null)); clusterService.addListener(this); } - private SecurityIndexManager(Client client, String indexName, int internalIndexFormat, String aliasName, String templateName, + private SecurityIndexManager(Client client, String indexName, int expectedIndexFormat, String aliasName, String templateName, State indexState) { this.client = client; this.indexName = indexName; - this.internalIndexFormat = internalIndexFormat; + this.expectedIndexFormat = expectedIndexFormat; this.aliasName = aliasName; this.templateName = templateName; this.indexState = indexState; } public SecurityIndexManager freeze() { - return new SecurityIndexManager(null, indexName, internalIndexFormat, aliasName, templateName, indexState); + return new SecurityIndexManager(null, indexName, expectedIndexFormat, aliasName, templateName, indexState); } public static List indexNames() { @@ -132,42 +131,44 @@ public boolean checkMappingVersion(Predicate requiredVersion) { return currentIndexState.mappingVersion == null || requiredVersion.test(currentIndexState.mappingVersion); } - public String indexName() { - return indexName; + public String aliasName() { + return aliasName; } - public boolean indexExists() { - return this.indexState.indexExists; + public boolean exists() { + return this.indexState.exists; } /** * Returns whether the index is on the current format if it exists. If the index does not exist * we treat the index as up to date as we expect it to be created with the current format. */ - public boolean isIndexUpToDate() { - return this.indexState.isIndexUpToDate; - } - - public boolean isIndexUpgraded() { - return this.indexState.isIndexUpgraded; + public boolean isUpToDate() { + return this.indexState.isUpToDate; } + /** + * Returns whether all primary shards are assigned. + */ public boolean isAvailable() { - return this.indexState.indexAvailable; + return this.indexState.available; } + /** + * Returns whether the index has the latest mapping. Mapping can change at any version. + */ public boolean isMappingUpToDate() { return this.indexState.mappingUpToDate; } public ElasticsearchException getUnavailableReason() { final State localState = this.indexState; - if (localState.indexAvailable) { - throw new IllegalStateException("caller must make sure to use a frozen state and check indexAvailable [" + indexName + "]"); + if (localState.available) { + throw new IllegalStateException("caller must make sure to use a frozen state and check isAvailable [" + indexName + "]"); } - if (localState.indexExists) { - return new UnavailableShardsException(null, "at least one primary shard for the security index [" + indexName + "] is unavailable"); + if (localState.exists) { + return new UnavailableShardsException(null, "at least one primary shard for the index [" + indexName + "] is unavailable"); } else { return new IndexNotFoundException(indexName); } @@ -191,19 +192,21 @@ public void clusterChanged(ClusterChangedEvent event) { return; } final State previousState = indexState; - final IndexMetaData indexMetaData = resolveConcreteIndex(indexName, event.state().metaData()); + // check for the index starting with the alias + final IndexMetaData indexMetaData = resolveConcreteIndex(aliasName, event.state().metaData()); final boolean indexExists = indexMetaData != null; + // if index does not exist it will be created with an up to date version + // if the version is newer that the latest expected, then a manual upgrade has been issued final boolean isIndexUpToDate = indexExists == false - || INDEX_FORMAT_SETTING.get(indexMetaData.getSettings()).intValue() >= internalIndexFormat; - final boolean isIndexUpgraded = indexExists == true - && INDEX_FORMAT_SETTING.get(indexMetaData.getSettings()).intValue() == internalIndexFormat + 1; + || INDEX_FORMAT_SETTING.get(indexMetaData.getSettings()).intValue() >= expectedIndexFormat; + // all primary shards allocated final boolean indexAvailable = checkIndexAvailable(event.state()); + // mapping can change in any version, unlike the "index.format" final boolean mappingIsUpToDate = indexExists == false || checkIndexMappingUpToDate(event.state()); final Version mappingVersion = oldestIndexMappingVersion(event.state()); final ClusterHealthStatus indexStatus = indexMetaData == null ? null : new ClusterIndexHealth(indexMetaData, event.state().getRoutingTable().index(indexMetaData.getIndex())).getStatus(); - final State newState = new State(indexExists, isIndexUpToDate, isIndexUpgraded, indexAvailable, mappingIsUpToDate, mappingVersion, - indexStatus); + final State newState = new State(indexExists, isIndexUpToDate, indexAvailable, mappingIsUpToDate, mappingVersion, indexStatus); this.indexState = newState; if (newState.equals(previousState) == false) { @@ -218,7 +221,7 @@ private boolean checkIndexAvailable(ClusterState state) { if (routingTable != null && routingTable.allPrimaryShardsActive()) { return true; } - logger.debug("Security index [{}] is not yet active", indexName); + logger.debug("Index [{}] is not yet active", indexName); return false; } @@ -226,7 +229,7 @@ private boolean checkIndexAvailable(ClusterState state) { * Returns the routing-table for this index, or null if the index does not exist. */ private IndexRoutingTable getIndexRoutingTable(ClusterState clusterState) { - IndexMetaData metaData = resolveConcreteIndex(indexName, clusterState.metaData()); + IndexMetaData metaData = resolveConcreteIndex(aliasName, clusterState.metaData()); if (metaData == null) { return null; } else { @@ -235,30 +238,29 @@ private IndexRoutingTable getIndexRoutingTable(ClusterState clusterState) { } private boolean checkIndexMappingUpToDate(ClusterState clusterState) { - return checkIndexMappingVersionMatches(indexName, clusterState, Version.CURRENT::equals); + return checkIndexMappingVersionMatches(aliasName, clusterState, Version.CURRENT::equals); } // public and static for testing - public static boolean checkIndexMappingVersionMatches(String indexName, ClusterState clusterState, Predicate predicate) { - return loadIndexMappingVersions(indexName, clusterState, logger).stream().allMatch(predicate); + public static boolean checkIndexMappingVersionMatches(String aliasName, ClusterState clusterState, Predicate predicate) { + return loadIndexMappingVersions(aliasName, clusterState, logger).stream().allMatch(predicate); } private Version oldestIndexMappingVersion(ClusterState clusterState) { - final Set versions = loadIndexMappingVersions(indexName, clusterState, logger); + final Set versions = loadIndexMappingVersions(aliasName, clusterState, logger); return versions.stream().min(Version::compareTo).orElse(null); } - private static Set loadIndexMappingVersions(String indexName, - ClusterState clusterState, Logger logger) { + private static Set loadIndexMappingVersions(String aliasName, ClusterState clusterState, Logger logger) { Set versions = new HashSet<>(); - IndexMetaData indexMetaData = resolveConcreteIndex(indexName, clusterState.metaData()); + IndexMetaData indexMetaData = resolveConcreteIndex(aliasName, clusterState.metaData()); if (indexMetaData != null) { for (Object object : indexMetaData.getMappings().values().toArray()) { MappingMetaData mappingMetaData = (MappingMetaData) object; if (mappingMetaData.type().equals(MapperService.DEFAULT_MAPPING)) { continue; } - versions.add(readMappingVersion(indexName, mappingMetaData, logger)); + versions.add(readMappingVersion(indexMetaData.getIndex().getName(), mappingMetaData, logger)); } } return versions; @@ -281,21 +283,17 @@ private static IndexMetaData resolveConcreteIndex(final String indexOrAliasName, return null; } - private static Version readMappingVersion(String indexName, MappingMetaData mappingMetaData, - Logger logger) { + private static Version readMappingVersion(String indexName, MappingMetaData mappingMetaData, Logger logger) { try { - Map meta = - (Map) mappingMetaData.sourceAsMap().get("_meta"); + Map meta = (Map) mappingMetaData.sourceAsMap().get("_meta"); if (meta == null) { logger.info("Missing _meta field in mapping [{}] of index [{}]", mappingMetaData.type(), indexName); throw new IllegalStateException("Cannot read security-version string in index " + indexName); } return Version.fromString((String) meta.get(SECURITY_VERSION_STRING)); } catch (ElasticsearchParseException e) { - logger.error(new ParameterizedMessage( - "Cannot parse the mapping for index [{}]", indexName), e); - throw new ElasticsearchException( - "Cannot parse the mapping for index [{}]", e, indexName); + logger.error(new ParameterizedMessage("Cannot parse the mapping for index [{}]", indexName), e); + throw new ElasticsearchException("Cannot parse the mapping for index [{}]", e, indexName); } } @@ -307,10 +305,9 @@ private static Version readMappingVersion(String indexName, MappingMetaData mapp */ public void checkIndexVersionThenExecute(final Consumer consumer, final Runnable andThen) { final State indexState = this.indexState; // use a local copy so all checks execute against the same state! - if (indexState.indexExists && indexState.isIndexUpToDate == false) { - consumer.accept(new IllegalStateException( - "Security index is not on the current version. Security features relying on the index will not be available until " + - "the upgrade API is run on the security index")); + if (indexState.exists && indexState.isUpToDate == false) { + consumer.accept(new IllegalStateException("Index [" + indexName + "] is not on the current version. " + + "Security features relying on the index will not be available until the upgrade API is run on the security index.")); } else { andThen.run(); } @@ -323,11 +320,10 @@ public void checkIndexVersionThenExecute(final Consumer consumer, fin public void prepareIndexIfNeededThenExecute(final Consumer consumer, final Runnable andThen) { final State indexState = this.indexState; // use a local copy so all checks execute against the same state! // TODO we should improve this so we don't fire off a bunch of requests to do the same thing (create or update mappings) - if (indexState.indexExists && indexState.isIndexUpToDate == false) { - consumer.accept(new IllegalStateException( - "Security index is not on the current version. Security features relying on the index will not be available until " + - "the upgrade API is run on the security index")); - } else if (indexState.indexExists == false) { + if (indexState.exists && indexState.isUpToDate == false) { + consumer.accept(new IllegalStateException("Index [" + indexName + "] is not on the current version. " + + "Security features relying on the index will not be available until the upgrade API is run on the security index")); + } else if (indexState.exists == false) { Tuple mappingAndSettings = loadMappingAndSettingsSourceFromTemplate(); CreateIndexRequest request = new CreateIndexRequest(indexName) .alias(new Alias(aliasName)) @@ -400,20 +396,18 @@ public static boolean isIndexDeleted(State previousState, State currentState) { * State of the security index. */ public static class State { - public final boolean indexExists; - public final boolean isIndexUpToDate; - public final boolean isIndexUpgraded; - public final boolean indexAvailable; + public final boolean exists; + public final boolean isUpToDate; + public final boolean available; public final boolean mappingUpToDate; public final Version mappingVersion; public final ClusterHealthStatus indexStatus; - public State(boolean indexExists, boolean isIndexUpToDate, boolean isIndexUpgraded, boolean indexAvailable, - boolean mappingUpToDate, Version mappingVersion, ClusterHealthStatus indexStatus) { - this.indexExists = indexExists; - this.isIndexUpToDate = isIndexUpToDate; - this.isIndexUpgraded = isIndexUpgraded; - this.indexAvailable = indexAvailable; + public State(boolean indexExists, boolean isIndexUpToDate, boolean indexAvailable, boolean mappingUpToDate, Version mappingVersion, + ClusterHealthStatus indexStatus) { + this.exists = indexExists; + this.isUpToDate = isIndexUpToDate; + this.available = indexAvailable; this.mappingUpToDate = mappingUpToDate; this.mappingVersion = mappingVersion; this.indexStatus = indexStatus; @@ -424,10 +418,9 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; State state = (State) o; - return indexExists == state.indexExists && - isIndexUpToDate == state.isIndexUpToDate && - isIndexUpgraded == state.isIndexUpgraded && - indexAvailable == state.indexAvailable && + return exists == state.exists && + isUpToDate == state.isUpToDate && + available == state.available && mappingUpToDate == state.mappingUpToDate && Objects.equals(mappingVersion, state.mappingVersion) && indexStatus == state.indexStatus; @@ -435,8 +428,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(indexExists, isIndexUpToDate, isIndexUpgraded, indexAvailable, mappingUpToDate, mappingVersion, - indexStatus); + return Objects.hash(exists, isUpToDate, available, mappingUpToDate, mappingVersion, indexStatus); } } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java index 66485f0505c5f..d72eaeaf8ae2f 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java @@ -194,7 +194,7 @@ void doExecute(Action action, Request request, ActionListener { @@ -1001,7 +1001,7 @@ public void testInvalidToken() throws Exception { public void testExpiredToken() throws Exception { when(securityIndex.isAvailable()).thenReturn(true); - when(securityIndex.indexExists()).thenReturn(true); + when(securityIndex.exists()).thenReturn(true); User user = new User("_username", "r1"); final Authentication expected = new Authentication(user, new RealmRef("realm", "custom", "node"), null); PlainActionFuture> tokenFuture = new PlainActionFuture<>(); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java index 47770288b1b66..4bf380a473ed1 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java @@ -120,7 +120,7 @@ public void setupClient() { runnable.run(); return null; }).when(securityIndex).checkIndexVersionThenExecute(any(Consumer.class), any(Runnable.class)); - when(securityIndex.indexExists()).thenReturn(true); + when(securityIndex.exists()).thenReturn(true); when(securityIndex.isAvailable()).thenReturn(true); this.clusterService = ClusterServiceUtils.createClusterService(threadPool); } @@ -377,7 +377,7 @@ public void testGetTokenWhenKeyCacheHasExpired() throws Exception { } public void testInvalidatedToken() throws Exception { - when(securityIndex.indexExists()).thenReturn(true); + when(securityIndex.exists()).thenReturn(true); TokenService tokenService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, securityIndex, clusterService); Authentication authentication = new Authentication(new User("joe", "admin"), new RealmRef("native_realm", "native", "node1"), null); @@ -542,18 +542,18 @@ public void testIndexNotAvailable() throws Exception { assertNull(future.get()); when(securityIndex.isAvailable()).thenReturn(false); - when(securityIndex.indexExists()).thenReturn(true); + when(securityIndex.exists()).thenReturn(true); future = new PlainActionFuture<>(); tokenService.getAndValidateToken(requestContext, future); assertNull(future.get()); - when(securityIndex.indexExists()).thenReturn(false); + when(securityIndex.exists()).thenReturn(false); future = new PlainActionFuture<>(); tokenService.getAndValidateToken(requestContext, future); assertNull(future.get()); when(securityIndex.isAvailable()).thenReturn(true); - when(securityIndex.indexExists()).thenReturn(true); + when(securityIndex.exists()).thenReturn(true); mockGetTokenFromId(token, false); future = new PlainActionFuture<>(); tokenService.getAndValidateToken(requestContext, future); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java index 175d26fc6a084..cda918b9dabca 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java @@ -236,9 +236,9 @@ private void respondToGetUserRequest(String username, SecureString password, Str private NativeUsersStore startNativeUsersStore() { SecurityIndexManager securityIndex = mock(SecurityIndexManager.class); when(securityIndex.isAvailable()).thenReturn(true); - when(securityIndex.indexExists()).thenReturn(true); + when(securityIndex.exists()).thenReturn(true); when(securityIndex.isMappingUpToDate()).thenReturn(true); - when(securityIndex.isIndexUpToDate()).thenReturn(true); + when(securityIndex.isUpToDate()).thenReturn(true); when(securityIndex.freeze()).thenReturn(securityIndex); doAnswer((i) -> { Runnable action = (Runnable) i.getArguments()[1]; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealmTests.java index 070ea855800f7..19f59486a9e09 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealmTests.java @@ -105,7 +105,7 @@ public void testAuthenticationDisabled() throws Throwable { Settings settings = Settings.builder().put(XPackSettings.RESERVED_REALM_ENABLED_SETTING.getKey(), false).build(); final boolean securityIndexExists = randomBoolean(); if (securityIndexExists) { - when(securityIndex.indexExists()).thenReturn(true); + when(securityIndex.exists()).thenReturn(true); } final ReservedRealm reservedRealm = new ReservedRealm(mock(Environment.class), settings, usersStore, @@ -137,7 +137,7 @@ private void verifySuccessfulAuthentication(boolean enabled) throws Exception { final SecureString newPassword = new SecureString("foobar".toCharArray()); // Mocked users store is initiated with default hashing algorithm final Hasher hasher = Hasher.resolve("bcrypt"); - when(securityIndex.indexExists()).thenReturn(true); + when(securityIndex.exists()).thenReturn(true); doAnswer((i) -> { ActionListener callback = (ActionListener) i.getArguments()[1]; callback.onResponse(new ReservedUserInfo(hasher.hash(newPassword), enabled, false)); @@ -163,7 +163,7 @@ private void verifySuccessfulAuthentication(boolean enabled) throws Exception { assertEquals(expectedUser, authenticated); assertThat(expectedUser.enabled(), is(enabled)); - verify(securityIndex, times(2)).indexExists(); + verify(securityIndex, times(2)).exists(); verify(usersStore, times(2)).getReservedUserInfo(eq(principal), any(ActionListener.class)); final ArgumentCaptor predicateCaptor = ArgumentCaptor.forClass(Predicate.class); verify(securityIndex, times(2)).checkMappingVersion(predicateCaptor.capture()); @@ -182,7 +182,7 @@ public void testLookup() throws Exception { reservedRealm.doLookupUser(principal, listener); final User user = listener.actionGet(); assertEquals(expectedUser, user); - verify(securityIndex).indexExists(); + verify(securityIndex).exists(); final ArgumentCaptor predicateCaptor = ArgumentCaptor.forClass(Predicate.class); verify(securityIndex).checkMappingVersion(predicateCaptor.capture()); @@ -216,7 +216,7 @@ public void testLookupThrows() throws Exception { new AnonymousUser(Settings.EMPTY), securityIndex, threadPool); final User expectedUser = randomReservedUser(true); final String principal = expectedUser.principal(); - when(securityIndex.indexExists()).thenReturn(true); + when(securityIndex.exists()).thenReturn(true); final RuntimeException e = new RuntimeException("store threw"); doAnswer((i) -> { ActionListener callback = (ActionListener) i.getArguments()[1]; @@ -229,7 +229,7 @@ public void testLookupThrows() throws Exception { ElasticsearchSecurityException securityException = expectThrows(ElasticsearchSecurityException.class, future::actionGet); assertThat(securityException.getMessage(), containsString("failed to lookup")); - verify(securityIndex).indexExists(); + verify(securityIndex).exists(); verify(usersStore).getReservedUserInfo(eq(principal), any(ActionListener.class)); final ArgumentCaptor predicateCaptor = ArgumentCaptor.forClass(Predicate.class); @@ -287,7 +287,7 @@ public void testGetUsersDisabled() { } public void testFailedAuthentication() throws Exception { - when(securityIndex.indexExists()).thenReturn(true); + when(securityIndex.exists()).thenReturn(true); SecureString password = new SecureString("password".toCharArray()); // Mocked users store is initiated with default hashing algorithm final Hasher hasher = Hasher.resolve("bcrypt"); @@ -322,7 +322,7 @@ public void testBootstrapElasticPasswordWorksOnceSecurityIndexExists() throws Ex MockSecureSettings mockSecureSettings = new MockSecureSettings(); mockSecureSettings.setString("bootstrap.password", "foobar"); Settings settings = Settings.builder().setSecureSettings(mockSecureSettings).build(); - when(securityIndex.indexExists()).thenReturn(true); + when(securityIndex.exists()).thenReturn(true); final ReservedRealm reservedRealm = new ReservedRealm(mock(Environment.class), settings, usersStore, new AnonymousUser(Settings.EMPTY), securityIndex, threadPool); @@ -344,7 +344,7 @@ public void testBootstrapElasticPasswordFailsOnceElasticUserExists() throws Exce MockSecureSettings mockSecureSettings = new MockSecureSettings(); mockSecureSettings.setString("bootstrap.password", "foobar"); Settings settings = Settings.builder().setSecureSettings(mockSecureSettings).build(); - when(securityIndex.indexExists()).thenReturn(true); + when(securityIndex.exists()).thenReturn(true); final ReservedRealm reservedRealm = new ReservedRealm(mock(Environment.class), settings, usersStore, new AnonymousUser(Settings.EMPTY), securityIndex, threadPool); @@ -373,7 +373,7 @@ public void testBootstrapElasticPasswordWorksBeforeSecurityIndexExists() throws MockSecureSettings mockSecureSettings = new MockSecureSettings(); mockSecureSettings.setString("bootstrap.password", "foobar"); Settings settings = Settings.builder().setSecureSettings(mockSecureSettings).build(); - when(securityIndex.indexExists()).thenReturn(false); + when(securityIndex.exists()).thenReturn(false); final ReservedRealm reservedRealm = new ReservedRealm(mock(Environment.class), settings, usersStore, new AnonymousUser(Settings.EMPTY), securityIndex, threadPool); @@ -391,7 +391,7 @@ public void testNonElasticUsersCannotUseBootstrapPasswordWhenSecurityIndexExists final String password = randomAlphaOfLengthBetween(8, 24); mockSecureSettings.setString("bootstrap.password", password); Settings settings = Settings.builder().setSecureSettings(mockSecureSettings).build(); - when(securityIndex.indexExists()).thenReturn(true); + when(securityIndex.exists()).thenReturn(true); final ReservedRealm reservedRealm = new ReservedRealm(mock(Environment.class), settings, usersStore, new AnonymousUser(Settings.EMPTY), securityIndex, threadPool); @@ -414,7 +414,7 @@ public void testNonElasticUsersCannotUseBootstrapPasswordWhenSecurityIndexDoesNo final String password = randomAlphaOfLengthBetween(8, 24); mockSecureSettings.setString("bootstrap.password", password); Settings settings = Settings.builder().setSecureSettings(mockSecureSettings).build(); - when(securityIndex.indexExists()).thenReturn(false); + when(securityIndex.exists()).thenReturn(false); final ReservedRealm reservedRealm = new ReservedRealm(mock(Environment.class), settings, usersStore, new AnonymousUser(Settings.EMPTY), securityIndex, threadPool); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java index 51bc1ad0ef3a1..373bd676b5b5a 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java @@ -92,7 +92,7 @@ void doExecute(Action action, Request request, ActionListener { assertThat(invocationOnMock.getArguments().length, equalTo(2)); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java index dd2d4d0e1aa33..d155f0a5716e3 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java @@ -96,7 +96,7 @@ public void testIndexWithUpToDateMappingAndTemplate() throws IOException { markShardsAvailable(clusterStateBuilder); manager.clusterChanged(event(clusterStateBuilder)); - assertThat(manager.indexExists(), Matchers.equalTo(true)); + assertThat(manager.exists(), Matchers.equalTo(true)); assertThat(manager.isAvailable(), Matchers.equalTo(true)); assertThat(manager.isMappingUpToDate(), Matchers.equalTo(true)); } @@ -198,13 +198,13 @@ public void testIndexOutOfDateListeners() throws Exception { AtomicBoolean upToDateChanged = new AtomicBoolean(); manager.addIndexStateListener((prev, current) -> { listenerCalled.set(true); - upToDateChanged.set(prev.isIndexUpToDate != current.isIndexUpToDate); + upToDateChanged.set(prev.isUpToDate != current.isUpToDate); }); - assertTrue(manager.isIndexUpToDate()); + assertTrue(manager.isUpToDate()); manager.clusterChanged(event(new ClusterState.Builder(CLUSTER_NAME))); assertFalse(listenerCalled.get()); - assertTrue(manager.isIndexUpToDate()); + assertTrue(manager.isUpToDate()); // index doesn't exist and now exists with wrong format ClusterState.Builder clusterStateBuilder = createClusterState(INDEX_NAME, TEMPLATE_NAME, @@ -213,14 +213,14 @@ public void testIndexOutOfDateListeners() throws Exception { manager.clusterChanged(event(clusterStateBuilder)); assertTrue(listenerCalled.get()); assertTrue(upToDateChanged.get()); - assertFalse(manager.isIndexUpToDate()); + assertFalse(manager.isUpToDate()); listenerCalled.set(false); assertFalse(listenerCalled.get()); manager.clusterChanged(event(new ClusterState.Builder(CLUSTER_NAME))); assertTrue(listenerCalled.get()); assertTrue(upToDateChanged.get()); - assertTrue(manager.isIndexUpToDate()); + assertTrue(manager.isUpToDate()); listenerCalled.set(false); // index doesn't exist and now exists with correct format @@ -229,17 +229,17 @@ public void testIndexOutOfDateListeners() throws Exception { manager.clusterChanged(event(clusterStateBuilder)); assertTrue(listenerCalled.get()); assertFalse(upToDateChanged.get()); - assertTrue(manager.isIndexUpToDate()); + assertTrue(manager.isUpToDate()); } private void assertInitialState() { - assertThat(manager.indexExists(), Matchers.equalTo(false)); + assertThat(manager.exists(), Matchers.equalTo(false)); assertThat(manager.isAvailable(), Matchers.equalTo(false)); assertThat(manager.isMappingUpToDate(), Matchers.equalTo(false)); } private void assertIndexUpToDateButNotAvailable() { - assertThat(manager.indexExists(), Matchers.equalTo(true)); + assertThat(manager.exists(), Matchers.equalTo(true)); assertThat(manager.isAvailable(), Matchers.equalTo(false)); assertThat(manager.isMappingUpToDate(), Matchers.equalTo(true)); } diff --git a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/InternalIndexReindexer.java b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/InternalIndexReindexer.java index 6ab920555bb0b..e8f2ab8c49cd4 100644 --- a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/InternalIndexReindexer.java +++ b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/InternalIndexReindexer.java @@ -32,11 +32,12 @@ /** * A component that performs the following upgrade procedure: *

- * - Check that all data and master nodes are running running the same version - * - Create a new index .{name}-6 + * - Check that all data and master nodes are running a compatible version (same minor) + * - Checks + * - Create a new index .{name}-{version} * - Make index .{name} read only - * - Reindex from .{name} to .{name}-6 with transform - * - Delete index .{name} and add alias .{name} to .{name}-6 + * - Reindex from .{name} to .{name}-{version} with transform + * - Delete index .{name} and add alias .{name} to .{name}-{version} */ public class InternalIndexReindexer { @@ -49,8 +50,7 @@ public class InternalIndexReindexer { private final BiConsumer> postUpgrade; public InternalIndexReindexer(Client client, ClusterService clusterService, int version, Script transformScript, String[] types, - Consumer> preUpgrade, - BiConsumer> postUpgrade) { + Consumer> preUpgrade, BiConsumer> postUpgrade) { this.client = client; this.clusterService = clusterService; this.transformScript = transformScript; @@ -60,10 +60,10 @@ public InternalIndexReindexer(Client client, ClusterService clusterService, int this.postUpgrade = postUpgrade; } - public void upgrade(TaskId task, String index, ClusterState clusterState, ActionListener listener) { + public void upgrade(TaskId task, String alias, ClusterState clusterState, ActionListener listener) { ParentTaskAssigningClient parentAwareClient = new ParentTaskAssigningClient(client, task); preUpgrade.accept(ActionListener.wrap( - t -> innerUpgrade(parentAwareClient, index, clusterState, ActionListener.wrap( + t -> innerUpgrade(parentAwareClient, alias, clusterState, ActionListener.wrap( response -> postUpgrade.accept(t, ActionListener.wrap( empty -> listener.onResponse(response), listener::onFailure @@ -73,30 +73,33 @@ public void upgrade(TaskId task, String index, ClusterState clusterState, Action listener::onFailure)); } - private void innerUpgrade(ParentTaskAssigningClient parentAwareClient, String index, ClusterState clusterState, + private void innerUpgrade(ParentTaskAssigningClient parentAwareClient, String alias, ClusterState clusterState, ActionListener listener) { - String newIndex = index + "-" + version; + final String newIndex = alias + "-" + version; + final String oldIndex = alias + "-" + (version - 1); try { checkMasterAndDataNodeVersion(clusterState); - parentAwareClient.admin().indices().prepareCreate(newIndex).execute(ActionListener.wrap(createIndexResponse -> - setReadOnlyBlock(index, ActionListener.wrap(setReadOnlyResponse -> - reindex(parentAwareClient, index, newIndex, ActionListener.wrap( - bulkByScrollResponse -> // Successful completion of reindexing - delete old index - removeReadOnlyBlock(parentAwareClient, index, ActionListener.wrap(unsetReadOnlyResponse -> - parentAwareClient.admin().indices().prepareAliases().removeIndex(index) - .addAlias(newIndex, index).execute(ActionListener.wrap(deleteIndexResponse -> - listener.onResponse(bulkByScrollResponse), listener::onFailure - )), listener::onFailure - )), - e -> // Something went wrong during reindexing - remove readonly flag and report the error - removeReadOnlyBlock(parentAwareClient, index, ActionListener.wrap(unsetReadOnlyResponse -> { - listener.onFailure(e); - }, e1 -> { - listener.onFailure(e); - })) - )), listener::onFailure - )), listener::onFailure - )); + checkAliasesAndIndices(clusterState, alias, oldIndex, newIndex); + setReadOnlyBlock(alias, ActionListener.wrap(setReadOnlyResponse -> + reindex(parentAwareClient, alias, newIndex, ActionListener.wrap( + bulkByScrollResponse -> // Successful completion of reindexing - delete old index + removeReadOnlyBlock(parentAwareClient, alias, ActionListener.wrap(unsetReadOnlyResponse -> + parentAwareClient.admin().indices().prepareAliases() + .removeIndex(oldIndex) + .removeAlias(oldIndex, alias) + .addAlias(newIndex, alias) + .execute(ActionListener.wrap(deleteIndexResponse -> + listener.onResponse(bulkByScrollResponse), + listener::onFailure)), + listener::onFailure)), + e -> // Something went wrong during reindexing - remove readonly flag and report the error + removeReadOnlyBlock(parentAwareClient, alias, ActionListener.wrap(unsetReadOnlyResponse -> { + listener.onFailure(e); + }, e1 -> { + listener.onFailure(e); + })) + )), listener::onFailure + )); } catch (Exception ex) { listener.onFailure(ex); } @@ -108,6 +111,27 @@ private void checkMasterAndDataNodeVersion(ClusterState clusterState) { } } + private void checkAliasesAndIndices(ClusterState clusterState, String alias, String oldIndex, String newIndex) { + final IndexMetaData newIndexMetadata = clusterState.metaData().index(newIndex); + if (newIndexMetadata == null) { + throw new IllegalStateException("Index [" + newIndex + "] does not exist, but is required for the upgrade."); + } + final IndexMetaData oldIndexMetadata = clusterState.metaData().index(oldIndex); + if (oldIndexMetadata == null) { + throw new IllegalStateException("Index [" + oldIndex + "] does not exist, but is required for the upgrade."); + } + final IndexMetaData aliasMetadata = clusterState.metaData().index(alias); + if (aliasMetadata == null) { + throw new IllegalStateException("Alias [" + alias + "] to upgrade does not exist."); + } + if (aliasMetadata.getAliases().size() != 1) { + throw new IllegalStateException("Alias [" + alias + "] does not point to exactly one index."); + } + if (false == aliasMetadata.getAliases().containsKey(oldIndex)) { + throw new IllegalStateException("Alias [" + alias + "] does not point to the [" + oldIndex + "] index."); + } + } + private void removeReadOnlyBlock(ParentTaskAssigningClient parentAwareClient, String index, ActionListener listener) { Settings settings = Settings.builder().put(IndexMetaData.INDEX_READ_ONLY_SETTING.getKey(), false).build(); diff --git a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/Upgrade.java b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/Upgrade.java index 985baeaf9ab3f..9cde2269c5cdd 100644 --- a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/Upgrade.java +++ b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/Upgrade.java @@ -44,7 +44,7 @@ public class Upgrade extends Plugin implements ActionPlugin { - public static final Version UPGRADE_INTRODUCED = Version.CURRENT.minimumCompatibilityVersion(); + public static final Version UPGRADE_INTRODUCED = Version.V_6_7_0; private final List> upgradeCheckFactories; From ec21cec2ad5bfd7d75b997f77d15cf5e294c5727 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Sat, 5 Jan 2019 12:31:34 +0200 Subject: [PATCH 08/19] WIP --- .../support/SecurityIndexManager.java | 2 +- .../xpack/upgrade/IndexUpgradeCheck.java | 4 +- .../xpack/upgrade/InternalIndexReindexer.java | 2 +- .../xpack/upgrade/InternalSplitReindexer.java | 216 ++++++++++++++++++ 4 files changed, 219 insertions(+), 5 deletions(-) create mode 100644 x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/InternalSplitReindexer.java diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java index bc25b44aef1e4..c450da3ef6fe8 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java @@ -155,7 +155,7 @@ public boolean isAvailable() { } /** - * Returns whether the index has the latest mapping. Mapping can change at any version. + * Returns whether the index has the mapping Mapping can change at any version. */ public boolean isMappingUpToDate() { return this.indexState.mappingUpToDate; diff --git a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/IndexUpgradeCheck.java b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/IndexUpgradeCheck.java index 62a2829b9258c..4537221a407f1 100644 --- a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/IndexUpgradeCheck.java +++ b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/IndexUpgradeCheck.java @@ -15,7 +15,6 @@ import org.elasticsearch.script.Script; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.transport.TransportResponse; -import org.elasticsearch.xpack.core.upgrade.IndexUpgradeCheckVersion; import java.util.function.BiConsumer; import java.util.function.Consumer; @@ -73,8 +72,7 @@ public IndexUpgradeCheck(String name, BiConsumer> postUpgrade) { this.name = name; this.actionRequired = actionRequired; - this.reindexer = new InternalIndexReindexer<>(client, clusterService, IndexUpgradeCheckVersion.UPRADE_VERSION, updateScript, - types, preUpgrade, postUpgrade); + this.reindexer = new InternalIndexReindexer<>(client, clusterService, 6, updateScript, types, preUpgrade, postUpgrade); } /** diff --git a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/InternalIndexReindexer.java b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/InternalIndexReindexer.java index e8f2ab8c49cd4..f624a7004b299 100644 --- a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/InternalIndexReindexer.java +++ b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/InternalIndexReindexer.java @@ -32,7 +32,7 @@ /** * A component that performs the following upgrade procedure: *

- * - Check that all data and master nodes are running a compatible version (same minor) + * - Check that all data and master nodes are running a compatible version * - Checks * - Create a new index .{name}-{version} * - Make index .{name} read only diff --git a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/InternalSplitReindexer.java b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/InternalSplitReindexer.java new file mode 100644 index 0000000000000..ba7dd4387313d --- /dev/null +++ b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/InternalSplitReindexer.java @@ -0,0 +1,216 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.upgrade; + +import org.apache.lucene.util.SetOnce; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.ParentTaskAssigningClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateUpdateTask; +import org.elasticsearch.cluster.block.ClusterBlocks; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.index.reindex.BulkByScrollResponse; +import org.elasticsearch.index.reindex.ReindexAction; +import org.elasticsearch.index.reindex.ReindexRequest; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.transport.TransportResponse; + +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.function.BiConsumer; +import java.util.function.Consumer; + +import static org.elasticsearch.index.IndexSettings.same; + +public class InternalSplitReindexer { + + private final Client client; + private final ClusterService clusterService; + private final String sourceIndex; + private final String sourceType; + private final Consumer> postUpgrade; + private final Destination[] destinations; + + public InternalSplitReindexer(Client client, ClusterService clusterService, String sourceIndex, String sourceType, + Consumer> postUpgrade, Destination... destinations) { + this.client = client; + this.clusterService = clusterService; + this.sourceIndex = sourceIndex; + this.sourceType = sourceType; + this.postUpgrade = postUpgrade; + this.destinations = destinations; + } + + public void upgrade(TaskId task, ActionListener listener) { + final ParentTaskAssigningClient parentAwareClient = new ParentTaskAssigningClient(client, task); + final ClusterState clusterState = clusterService.state(); + final CountDownLatch countDown = new CountDownLatch(destinations.length); + final List reindexExceptions = new CopyOnWriteArrayList<>(); + setReadOnlyBlock(sourceIndex, ActionListener.wrap(setReadOnlyResponse -> { + for (Destination destination : destinations) { + reindex(parentAwareClient, sourceIndex, sourceType, destination, ActionListener.wrap(bulkByScrollResponse -> { + countDown.countDown(); + destination.postUpgrade + if (co) + }, e -> { + countDown.countDown(); + reindexExceptions.add(e); + })); + } + }, listener::onFailure)); + + innerUpgrade(parentAwareClient, ActionListener.wrap( + response -> postUpgrade.accept(ActionListener.wrap( + empty -> removeReadOnlyBlock(parentAwareClient, sourceIndex, A listener.onResponse(response), listener::onFailure)), + listener::onFailure)); + } + + private void innerUpgrade(ParentTaskAssigningClient parentAwareClient, ActionListener listener) { + final ClusterState clusterState = clusterService.state(); + final CountDownLatch countDown = new CountDownLatch(destinations.length); + final SetOnce exception = new SetOnce<>(); + try { + setReadOnlyBlock(sourceIndex, ActionListener.wrap(setReadOnlyResponse -> { + for (Destination destination : destinations) { + reindex(parentAwareClient, sourceIndex, sourceType, destination, ActionListener.wrap(bulkByScrollResponse -> { + countDown.countDown(); + }, e -> { + countDown.countDown(); + })); + } + }, e -> listener.onFailure(e))); + } catch (Exception ex) { + listener.onFailure(ex); + } + + try { + final ClusterState clusterState = clusterService.state(); + final CountDownLatch countDown = new CountDownLatch(destinations.length); + checkMasterAndDataNodeVersion(clusterState); + for (Destination destination : destinations) { + setReadOnlyBlock(sourceIndex, ActionListener.wrap(setReadOnlyResponse -> + reindex(parentAwareClient, sourceIndex, sourceType, destination, ActionListener.wrap(bulkByScrollResponse -> + // Successful completion of reindexing + removeReadOnlyBlock(parentAwareClient, sourceIndex, ActionListener.wrap(unsetReadOnlyResponse -> + parentAwareClient.admin().indices().prepareAliases() + .removeIndex(oldIndex) + .removeAlias(oldIndex, alias) + .addAlias(newIndex, alias) + .execute(ActionListener.wrap(deleteIndexResponse -> + listener.onResponse(bulkByScrollResponse), + listener::onFailure)), + listener::onFailure)), + e -> // Something went wrong during reindexing - remove readonly flag and report the error + removeReadOnlyBlock(parentAwareClient, sourceIndex, ActionListener.wrap(unsetReadOnlyResponse -> { + listener.onFailure(e); + }, e1 -> { + listener.onFailure(e); + })) + )), + listener::onFailure + )); + + } + } catch (Exception ex) { + listener.onFailure(ex); + } + } + + private void checkMasterAndDataNodeVersion(ClusterState clusterState) { + if (clusterState.nodes().getMinNodeVersion().before(Upgrade.UPGRADE_INTRODUCED)) { + throw new IllegalStateException("All nodes should have at least version [" + Upgrade.UPGRADE_INTRODUCED + "] to upgrade"); + } + } + + private void removeReadOnlyBlock(ParentTaskAssigningClient parentAwareClient, String index, + ActionListener listener) { + Settings settings = Settings.builder().put(IndexMetaData.INDEX_READ_ONLY_SETTING.getKey(), false).build(); + parentAwareClient.admin().indices().prepareUpdateSettings(index).setSettings(settings).execute(listener); + } + + private void reindex(ParentTaskAssigningClient parentAwareClient, String sourceIndex, String sourceType, Destination destination, + ActionListener listener) { + ReindexRequest reindexRequest = new ReindexRequest(); + reindexRequest.setSourceIndices(sourceIndex); + reindexRequest.setSourceDocTypes(sourceType); + reindexRequest.setDestIndex(destination.index); + reindexRequest.setRefresh(true); + reindexRequest.setSourceQuery(destination.query); + parentAwareClient.execute(ReindexAction.INSTANCE, reindexRequest, listener); + } + + /** + * Makes the index readonly if it's not set as a readonly yet + */ + private void setReadOnlyBlock(String index, ActionListener listener) { + clusterService.submitStateUpdateTask("lock-index-for-upgrade", new ClusterStateUpdateTask() { + + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + final IndexMetaData indexMetaData = currentState.metaData().index(index); + if (indexMetaData == null) { + throw new IndexNotFoundException(index); + } + + if (indexMetaData.getState() != IndexMetaData.State.OPEN) { + throw new IllegalStateException("unable to upgrade a closed index[" + index + "]"); + } + if (currentState.blocks().hasIndexBlock(index, IndexMetaData.INDEX_READ_ONLY_BLOCK)) { + throw new IllegalStateException("unable to upgrade a read-only index[" + index + "]"); + } + + final Settings indexSettingsBuilder = + Settings.builder() + .put(indexMetaData.getSettings()) + .put(IndexMetaData.INDEX_READ_ONLY_SETTING.getKey(), true) + .build(); + final IndexMetaData.Builder builder = IndexMetaData.builder(indexMetaData).settings(indexSettingsBuilder); + assert same(indexMetaData.getSettings(), indexSettingsBuilder) == false; + builder.settingsVersion(1 + builder.settingsVersion()); + + MetaData.Builder metaDataBuilder = MetaData.builder(currentState.metaData()).put(builder); + + ClusterBlocks.Builder blocks = ClusterBlocks.builder().blocks(currentState.blocks()) + .addIndexBlock(index, IndexMetaData.INDEX_READ_ONLY_BLOCK); + + return ClusterState.builder(currentState).metaData(metaDataBuilder).blocks(blocks).build(); + } + + @Override + public void onFailure(String source, Exception e) { + listener.onFailure(e); + } + + @Override + public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { + listener.onResponse(TransportResponse.Empty.INSTANCE); + } + }); + } + + static class Destination { + + private final String index; + private final QueryBuilder query; + private final Consumer> postUpgrade; + + Destination(String index, QueryBuilder query, Consumer> postUpgrade) { + this.index = index; + this.query = query; + this.postUpgrade = postUpgrade; + } + } + +} From 441b971bf8c191b57f389f8d886a8ce7e8916215 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Sat, 5 Jan 2019 13:15:51 +0200 Subject: [PATCH 09/19] WIP --- .../new-security-index-template.json | 159 +++++++++++++ .../xpack/upgrade/InternalIndexReindexer.java | 82 +++---- .../xpack/upgrade/InternalSplitReindexer.java | 216 ------------------ .../upgrade/SecurityIndexUpgradeCheck.java | 53 +++++ 4 files changed, 241 insertions(+), 269 deletions(-) create mode 100644 x-pack/plugin/core/src/main/resources/new-security-index-template.json delete mode 100644 x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/InternalSplitReindexer.java create mode 100644 x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/SecurityIndexUpgradeCheck.java diff --git a/x-pack/plugin/core/src/main/resources/new-security-index-template.json b/x-pack/plugin/core/src/main/resources/new-security-index-template.json new file mode 100644 index 0000000000000..9e756b0b8b605 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/new-security-index-template.json @@ -0,0 +1,159 @@ +{ + "index_patterns" : [ ".security-*" ], + "order" : 1000, + "settings" : { + "number_of_shards" : 1, + "number_of_replicas" : 0, + "auto_expand_replicas" : "0-1", + "index.priority": 1000, + "index.format": 6, + "analysis" : { + "filter" : { + "email" : { + "type" : "pattern_capture", + "preserve_original" : true, + "patterns" : [ + "([^@]+)", + "(\\p{L}+)", + "(\\d+)", + "@(.+)" + ] + } + }, + "analyzer" : { + "email" : { + "tokenizer" : "uax_url_email", + "filter" : [ + "email", + "lowercase", + "unique" + ] + } + } + } + }, + "mappings" : { + "doc" : { + "_meta": { + "security-version": "${security.template.version}" + }, + "dynamic" : "strict", + "properties" : { + "username" : { + "type" : "keyword" + }, + "roles" : { + "type" : "keyword" + }, + "password" : { + "type" : "keyword", + "index" : false, + "doc_values": false + }, + "full_name" : { + "type" : "text" + }, + "email" : { + "type" : "text", + "analyzer" : "email" + }, + "metadata" : { + "type" : "object", + "dynamic" : true + }, + "enabled": { + "type": "boolean" + }, + "cluster" : { + "type" : "keyword" + }, + "indices" : { + "type" : "object", + "properties" : { + "field_security" : { + "properties" : { + "grant": { + "type": "keyword" + }, + "except": { + "type": "keyword" + } + } + }, + "names" : { + "type" : "keyword" + }, + "privileges" : { + "type" : "keyword" + }, + "query" : { + "type" : "keyword" + } + } + }, + "applications": { + "type": "object", + "properties": { + "application": { + "type": "keyword" + }, + "privileges": { + "type": "keyword" + }, + "resources": { + "type": "keyword" + } + } + }, + "application" : { + "type" : "keyword" + }, + "global": { + "type": "object", + "properties": { + "application": { + "type": "object", + "properties": { + "manage": { + "type": "object", + "properties": { + "applications": { + "type": "keyword" + } + } + } + } + } + } + }, + "name" : { + "type" : "keyword" + }, + "run_as" : { + "type" : "keyword" + }, + "doc_type" : { + "type" : "keyword" + }, + "type" : { + "type" : "keyword" + }, + "actions" : { + "type" : "keyword" + }, + "expiration_time" : { + "type" : "date", + "format" : "epoch_millis" + }, + "creation_time" : { + "type" : "date", + "format" : "epoch_millis" + }, + "rules" : { + "type" : "object", + "dynamic" : true + } + } + } + } +} diff --git a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/InternalIndexReindexer.java b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/InternalIndexReindexer.java index f624a7004b299..6ab920555bb0b 100644 --- a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/InternalIndexReindexer.java +++ b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/InternalIndexReindexer.java @@ -32,12 +32,11 @@ /** * A component that performs the following upgrade procedure: *

- * - Check that all data and master nodes are running a compatible version - * - Checks - * - Create a new index .{name}-{version} + * - Check that all data and master nodes are running running the same version + * - Create a new index .{name}-6 * - Make index .{name} read only - * - Reindex from .{name} to .{name}-{version} with transform - * - Delete index .{name} and add alias .{name} to .{name}-{version} + * - Reindex from .{name} to .{name}-6 with transform + * - Delete index .{name} and add alias .{name} to .{name}-6 */ public class InternalIndexReindexer { @@ -50,7 +49,8 @@ public class InternalIndexReindexer { private final BiConsumer> postUpgrade; public InternalIndexReindexer(Client client, ClusterService clusterService, int version, Script transformScript, String[] types, - Consumer> preUpgrade, BiConsumer> postUpgrade) { + Consumer> preUpgrade, + BiConsumer> postUpgrade) { this.client = client; this.clusterService = clusterService; this.transformScript = transformScript; @@ -60,10 +60,10 @@ public InternalIndexReindexer(Client client, ClusterService clusterService, int this.postUpgrade = postUpgrade; } - public void upgrade(TaskId task, String alias, ClusterState clusterState, ActionListener listener) { + public void upgrade(TaskId task, String index, ClusterState clusterState, ActionListener listener) { ParentTaskAssigningClient parentAwareClient = new ParentTaskAssigningClient(client, task); preUpgrade.accept(ActionListener.wrap( - t -> innerUpgrade(parentAwareClient, alias, clusterState, ActionListener.wrap( + t -> innerUpgrade(parentAwareClient, index, clusterState, ActionListener.wrap( response -> postUpgrade.accept(t, ActionListener.wrap( empty -> listener.onResponse(response), listener::onFailure @@ -73,33 +73,30 @@ public void upgrade(TaskId task, String alias, ClusterState clusterState, Action listener::onFailure)); } - private void innerUpgrade(ParentTaskAssigningClient parentAwareClient, String alias, ClusterState clusterState, + private void innerUpgrade(ParentTaskAssigningClient parentAwareClient, String index, ClusterState clusterState, ActionListener listener) { - final String newIndex = alias + "-" + version; - final String oldIndex = alias + "-" + (version - 1); + String newIndex = index + "-" + version; try { checkMasterAndDataNodeVersion(clusterState); - checkAliasesAndIndices(clusterState, alias, oldIndex, newIndex); - setReadOnlyBlock(alias, ActionListener.wrap(setReadOnlyResponse -> - reindex(parentAwareClient, alias, newIndex, ActionListener.wrap( - bulkByScrollResponse -> // Successful completion of reindexing - delete old index - removeReadOnlyBlock(parentAwareClient, alias, ActionListener.wrap(unsetReadOnlyResponse -> - parentAwareClient.admin().indices().prepareAliases() - .removeIndex(oldIndex) - .removeAlias(oldIndex, alias) - .addAlias(newIndex, alias) - .execute(ActionListener.wrap(deleteIndexResponse -> - listener.onResponse(bulkByScrollResponse), - listener::onFailure)), - listener::onFailure)), - e -> // Something went wrong during reindexing - remove readonly flag and report the error - removeReadOnlyBlock(parentAwareClient, alias, ActionListener.wrap(unsetReadOnlyResponse -> { - listener.onFailure(e); - }, e1 -> { - listener.onFailure(e); - })) - )), listener::onFailure - )); + parentAwareClient.admin().indices().prepareCreate(newIndex).execute(ActionListener.wrap(createIndexResponse -> + setReadOnlyBlock(index, ActionListener.wrap(setReadOnlyResponse -> + reindex(parentAwareClient, index, newIndex, ActionListener.wrap( + bulkByScrollResponse -> // Successful completion of reindexing - delete old index + removeReadOnlyBlock(parentAwareClient, index, ActionListener.wrap(unsetReadOnlyResponse -> + parentAwareClient.admin().indices().prepareAliases().removeIndex(index) + .addAlias(newIndex, index).execute(ActionListener.wrap(deleteIndexResponse -> + listener.onResponse(bulkByScrollResponse), listener::onFailure + )), listener::onFailure + )), + e -> // Something went wrong during reindexing - remove readonly flag and report the error + removeReadOnlyBlock(parentAwareClient, index, ActionListener.wrap(unsetReadOnlyResponse -> { + listener.onFailure(e); + }, e1 -> { + listener.onFailure(e); + })) + )), listener::onFailure + )), listener::onFailure + )); } catch (Exception ex) { listener.onFailure(ex); } @@ -111,27 +108,6 @@ private void checkMasterAndDataNodeVersion(ClusterState clusterState) { } } - private void checkAliasesAndIndices(ClusterState clusterState, String alias, String oldIndex, String newIndex) { - final IndexMetaData newIndexMetadata = clusterState.metaData().index(newIndex); - if (newIndexMetadata == null) { - throw new IllegalStateException("Index [" + newIndex + "] does not exist, but is required for the upgrade."); - } - final IndexMetaData oldIndexMetadata = clusterState.metaData().index(oldIndex); - if (oldIndexMetadata == null) { - throw new IllegalStateException("Index [" + oldIndex + "] does not exist, but is required for the upgrade."); - } - final IndexMetaData aliasMetadata = clusterState.metaData().index(alias); - if (aliasMetadata == null) { - throw new IllegalStateException("Alias [" + alias + "] to upgrade does not exist."); - } - if (aliasMetadata.getAliases().size() != 1) { - throw new IllegalStateException("Alias [" + alias + "] does not point to exactly one index."); - } - if (false == aliasMetadata.getAliases().containsKey(oldIndex)) { - throw new IllegalStateException("Alias [" + alias + "] does not point to the [" + oldIndex + "] index."); - } - } - private void removeReadOnlyBlock(ParentTaskAssigningClient parentAwareClient, String index, ActionListener listener) { Settings settings = Settings.builder().put(IndexMetaData.INDEX_READ_ONLY_SETTING.getKey(), false).build(); diff --git a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/InternalSplitReindexer.java b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/InternalSplitReindexer.java deleted file mode 100644 index ba7dd4387313d..0000000000000 --- a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/InternalSplitReindexer.java +++ /dev/null @@ -1,216 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.upgrade; - -import org.apache.lucene.util.SetOnce; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.support.master.AcknowledgedResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.client.ParentTaskAssigningClient; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.ClusterStateUpdateTask; -import org.elasticsearch.cluster.block.ClusterBlocks; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.IndexNotFoundException; -import org.elasticsearch.index.reindex.BulkByScrollResponse; -import org.elasticsearch.index.reindex.ReindexAction; -import org.elasticsearch.index.reindex.ReindexRequest; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.tasks.TaskId; -import org.elasticsearch.transport.TransportResponse; - -import java.util.List; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.CountDownLatch; -import java.util.function.BiConsumer; -import java.util.function.Consumer; - -import static org.elasticsearch.index.IndexSettings.same; - -public class InternalSplitReindexer { - - private final Client client; - private final ClusterService clusterService; - private final String sourceIndex; - private final String sourceType; - private final Consumer> postUpgrade; - private final Destination[] destinations; - - public InternalSplitReindexer(Client client, ClusterService clusterService, String sourceIndex, String sourceType, - Consumer> postUpgrade, Destination... destinations) { - this.client = client; - this.clusterService = clusterService; - this.sourceIndex = sourceIndex; - this.sourceType = sourceType; - this.postUpgrade = postUpgrade; - this.destinations = destinations; - } - - public void upgrade(TaskId task, ActionListener listener) { - final ParentTaskAssigningClient parentAwareClient = new ParentTaskAssigningClient(client, task); - final ClusterState clusterState = clusterService.state(); - final CountDownLatch countDown = new CountDownLatch(destinations.length); - final List reindexExceptions = new CopyOnWriteArrayList<>(); - setReadOnlyBlock(sourceIndex, ActionListener.wrap(setReadOnlyResponse -> { - for (Destination destination : destinations) { - reindex(parentAwareClient, sourceIndex, sourceType, destination, ActionListener.wrap(bulkByScrollResponse -> { - countDown.countDown(); - destination.postUpgrade - if (co) - }, e -> { - countDown.countDown(); - reindexExceptions.add(e); - })); - } - }, listener::onFailure)); - - innerUpgrade(parentAwareClient, ActionListener.wrap( - response -> postUpgrade.accept(ActionListener.wrap( - empty -> removeReadOnlyBlock(parentAwareClient, sourceIndex, A listener.onResponse(response), listener::onFailure)), - listener::onFailure)); - } - - private void innerUpgrade(ParentTaskAssigningClient parentAwareClient, ActionListener listener) { - final ClusterState clusterState = clusterService.state(); - final CountDownLatch countDown = new CountDownLatch(destinations.length); - final SetOnce exception = new SetOnce<>(); - try { - setReadOnlyBlock(sourceIndex, ActionListener.wrap(setReadOnlyResponse -> { - for (Destination destination : destinations) { - reindex(parentAwareClient, sourceIndex, sourceType, destination, ActionListener.wrap(bulkByScrollResponse -> { - countDown.countDown(); - }, e -> { - countDown.countDown(); - })); - } - }, e -> listener.onFailure(e))); - } catch (Exception ex) { - listener.onFailure(ex); - } - - try { - final ClusterState clusterState = clusterService.state(); - final CountDownLatch countDown = new CountDownLatch(destinations.length); - checkMasterAndDataNodeVersion(clusterState); - for (Destination destination : destinations) { - setReadOnlyBlock(sourceIndex, ActionListener.wrap(setReadOnlyResponse -> - reindex(parentAwareClient, sourceIndex, sourceType, destination, ActionListener.wrap(bulkByScrollResponse -> - // Successful completion of reindexing - removeReadOnlyBlock(parentAwareClient, sourceIndex, ActionListener.wrap(unsetReadOnlyResponse -> - parentAwareClient.admin().indices().prepareAliases() - .removeIndex(oldIndex) - .removeAlias(oldIndex, alias) - .addAlias(newIndex, alias) - .execute(ActionListener.wrap(deleteIndexResponse -> - listener.onResponse(bulkByScrollResponse), - listener::onFailure)), - listener::onFailure)), - e -> // Something went wrong during reindexing - remove readonly flag and report the error - removeReadOnlyBlock(parentAwareClient, sourceIndex, ActionListener.wrap(unsetReadOnlyResponse -> { - listener.onFailure(e); - }, e1 -> { - listener.onFailure(e); - })) - )), - listener::onFailure - )); - - } - } catch (Exception ex) { - listener.onFailure(ex); - } - } - - private void checkMasterAndDataNodeVersion(ClusterState clusterState) { - if (clusterState.nodes().getMinNodeVersion().before(Upgrade.UPGRADE_INTRODUCED)) { - throw new IllegalStateException("All nodes should have at least version [" + Upgrade.UPGRADE_INTRODUCED + "] to upgrade"); - } - } - - private void removeReadOnlyBlock(ParentTaskAssigningClient parentAwareClient, String index, - ActionListener listener) { - Settings settings = Settings.builder().put(IndexMetaData.INDEX_READ_ONLY_SETTING.getKey(), false).build(); - parentAwareClient.admin().indices().prepareUpdateSettings(index).setSettings(settings).execute(listener); - } - - private void reindex(ParentTaskAssigningClient parentAwareClient, String sourceIndex, String sourceType, Destination destination, - ActionListener listener) { - ReindexRequest reindexRequest = new ReindexRequest(); - reindexRequest.setSourceIndices(sourceIndex); - reindexRequest.setSourceDocTypes(sourceType); - reindexRequest.setDestIndex(destination.index); - reindexRequest.setRefresh(true); - reindexRequest.setSourceQuery(destination.query); - parentAwareClient.execute(ReindexAction.INSTANCE, reindexRequest, listener); - } - - /** - * Makes the index readonly if it's not set as a readonly yet - */ - private void setReadOnlyBlock(String index, ActionListener listener) { - clusterService.submitStateUpdateTask("lock-index-for-upgrade", new ClusterStateUpdateTask() { - - @Override - public ClusterState execute(ClusterState currentState) throws Exception { - final IndexMetaData indexMetaData = currentState.metaData().index(index); - if (indexMetaData == null) { - throw new IndexNotFoundException(index); - } - - if (indexMetaData.getState() != IndexMetaData.State.OPEN) { - throw new IllegalStateException("unable to upgrade a closed index[" + index + "]"); - } - if (currentState.blocks().hasIndexBlock(index, IndexMetaData.INDEX_READ_ONLY_BLOCK)) { - throw new IllegalStateException("unable to upgrade a read-only index[" + index + "]"); - } - - final Settings indexSettingsBuilder = - Settings.builder() - .put(indexMetaData.getSettings()) - .put(IndexMetaData.INDEX_READ_ONLY_SETTING.getKey(), true) - .build(); - final IndexMetaData.Builder builder = IndexMetaData.builder(indexMetaData).settings(indexSettingsBuilder); - assert same(indexMetaData.getSettings(), indexSettingsBuilder) == false; - builder.settingsVersion(1 + builder.settingsVersion()); - - MetaData.Builder metaDataBuilder = MetaData.builder(currentState.metaData()).put(builder); - - ClusterBlocks.Builder blocks = ClusterBlocks.builder().blocks(currentState.blocks()) - .addIndexBlock(index, IndexMetaData.INDEX_READ_ONLY_BLOCK); - - return ClusterState.builder(currentState).metaData(metaDataBuilder).blocks(blocks).build(); - } - - @Override - public void onFailure(String source, Exception e) { - listener.onFailure(e); - } - - @Override - public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { - listener.onResponse(TransportResponse.Empty.INSTANCE); - } - }); - } - - static class Destination { - - private final String index; - private final QueryBuilder query; - private final Consumer> postUpgrade; - - Destination(String index, QueryBuilder query, Consumer> postUpgrade) { - this.index = index; - this.query = query; - this.postUpgrade = postUpgrade; - } - } - -} diff --git a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/SecurityIndexUpgradeCheck.java b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/SecurityIndexUpgradeCheck.java new file mode 100644 index 0000000000000..5d0922f798627 --- /dev/null +++ b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/SecurityIndexUpgradeCheck.java @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.upgrade; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.index.reindex.BulkByScrollResponse; +import org.elasticsearch.protocol.xpack.migration.UpgradeActionRequired; +import org.elasticsearch.tasks.TaskId; + +public class SecurityIndexUpgradeCheck { + + private final String name; + + public SecurityIndexUpgradeCheck(String name) { + this.name = name; + } + + /** + * Returns the name of the check + */ + public String getName() { + return name; + } + + /** + * This method is called by Upgrade API to verify if upgrade or reindex for this index is required + * + * @param indexMetaData index metadata + * @return required action or UpgradeActionRequired.NOT_APPLICABLE if this check cannot be performed on the index + */ + public UpgradeActionRequired actionRequired(IndexMetaData indexMetaData) { + return actionRequired.apply(indexMetaData); + } + + /** + * Perform the index upgrade + * + * @param task the task that executes the upgrade operation + * @param indexMetaData index metadata + * @param state current cluster state + * @param listener the listener that should be called upon completion of the upgrade + */ + public void upgrade(TaskId task, IndexMetaData indexMetaData, ClusterState state, + ActionListener listener) { + reindexer.upgrade(task, indexMetaData.getIndex().getName(), state, listener); + } +} From 24dafb3eada7f4513b46af2235d2c7d842831071 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Sun, 6 Jan 2019 02:52:31 +0200 Subject: [PATCH 10/19] Wonderfully --- x-pack/plugin/upgrade/build.gradle | 4 + .../upgrade/SecurityIndexUpgradeCheck.java | 184 +++++++++++++++++- 2 files changed, 183 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/upgrade/build.gradle b/x-pack/plugin/upgrade/build.gradle index 25a39168dbc09..0890520c3c04b 100644 --- a/x-pack/plugin/upgrade/build.gradle +++ b/x-pack/plugin/upgrade/build.gradle @@ -18,6 +18,10 @@ dependencies { testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') } +sourceSets.main.resources { + srcDir '../core/src/main/resources' +} + compileJava.options.compilerArgs << "-Xlint:-deprecation,-rawtypes,-serial,-try,-unchecked" compileTestJava.options.compilerArgs << "-Xlint:-deprecation,-rawtypes,-serial,-try,-unchecked" diff --git a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/SecurityIndexUpgradeCheck.java b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/SecurityIndexUpgradeCheck.java index 5d0922f798627..0624eac3fb97c 100644 --- a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/SecurityIndexUpgradeCheck.java +++ b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/SecurityIndexUpgradeCheck.java @@ -6,19 +6,63 @@ package org.elasticsearch.xpack.upgrade; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; +import org.elasticsearch.action.support.ActiveShardCount; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.ParentTaskAssigningClient; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateUpdateTask; +import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.reindex.BulkByScrollResponse; +import org.elasticsearch.index.reindex.ReindexAction; +import org.elasticsearch.index.reindex.ReindexRequest; import org.elasticsearch.protocol.xpack.migration.UpgradeActionRequired; import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.transport.TransportResponse; +import org.elasticsearch.xpack.core.template.TemplateUtils; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.function.Consumer; +import java.util.regex.Pattern; + +import static org.elasticsearch.index.IndexSettings.same; public class SecurityIndexUpgradeCheck { + private static final String SECURITY_ALIAS_NAME = ".security"; + private static final String TEMPLATE_VERSION_PATTERN = Pattern.quote("${security.template.version}"); + private static final int INTERNAL_SECURITY_INDEX_FORMAT = 6; + private static final String INTERNAL_SECURITY_INDEX = ".security-" + INTERNAL_SECURITY_INDEX_FORMAT; + private static final String NEW_INTERNAL_SECURITY_INDEX = ".security-" + (INTERNAL_SECURITY_INDEX_FORMAT + 1); + private static final String NEW_SECURITY_TEMPLATE_NAME = "new-security-index-template"; + + private static final String SECURITY_TOKENS_ALIAS_NAME = ".security-tokens"; + private static final int INTERNAL_SECURITY_TOKENS_INDEX_FORMAT = 7; + private static final String INTERNAL_SECURITY_TOKENS_INDEX = ".security-tokens-" + INTERNAL_SECURITY_TOKENS_INDEX_FORMAT; + private static final String SECURITY_TOKENS_TEMPLATE_NAME = "security-tokens-index-template"; + private final String name; + private final Client client; + private final ClusterService clusterService; - public SecurityIndexUpgradeCheck(String name) { + public SecurityIndexUpgradeCheck(String name, Client client, ClusterService clusterService) { this.name = name; + this.client = client; + this.clusterService = clusterService; } /** @@ -35,7 +79,12 @@ public String getName() { * @return required action or UpgradeActionRequired.NOT_APPLICABLE if this check cannot be performed on the index */ public UpgradeActionRequired actionRequired(IndexMetaData indexMetaData) { - return actionRequired.apply(indexMetaData); + if (indexMetaData.getIndex().getName().equals(INTERNAL_SECURITY_INDEX)) { + // no need to check the "index.format" setting as the name encodes the version + return UpgradeActionRequired.UPGRADE; + } else { + return UpgradeActionRequired.NOT_APPLICABLE; + } } /** @@ -46,8 +95,133 @@ public UpgradeActionRequired actionRequired(IndexMetaData indexMetaData) { * @param state current cluster state * @param listener the listener that should be called upon completion of the upgrade */ - public void upgrade(TaskId task, IndexMetaData indexMetaData, ClusterState state, - ActionListener listener) { - reindexer.upgrade(task, indexMetaData.getIndex().getName(), state, listener); + public void upgrade(TaskId task, IndexMetaData indexMetaData, ClusterState state, ActionListener listener) { + final ParentTaskAssigningClient parentAwareClient = new ParentTaskAssigningClient(client, task); + final BoolQueryBuilder tokensQuery = QueryBuilders.boolQuery().filter(QueryBuilders.termQuery("doc_type", "token")); + final BoolQueryBuilder nonTokensQuery = QueryBuilders.boolQuery() + .filter(QueryBuilders.boolQuery().mustNot(QueryBuilders.termQuery("doc_type", "token"))); + final Consumer removeReadOnlyConsumer = e -> removeReadOnlyBlock(parentAwareClient, INTERNAL_SECURITY_INDEX, + ActionListener.wrap(unsetReadOnlyResponse -> listener.onFailure(e), e1 -> listener.onFailure(e))); + // create the two new indices, but without their corresponding aliases + createNewSecurityIndices(parentAwareClient, ActionListener.wrap(createIndicesResponse -> { + // set a read-only block on the original .security-6 index + setReadOnlyBlock(INTERNAL_SECURITY_INDEX, ActionListener.wrap(setReadOnlyResponse -> { + // firstly, reindex all the tokens to the new tokens index + reindex(parentAwareClient, INTERNAL_SECURITY_INDEX, INTERNAL_SECURITY_TOKENS_INDEX, tokensQuery, + ActionListener.wrap(reindexTokensResponse -> { + // create the alias pointing to the new tokens index; now the new tokens index is writable anew + parentAwareClient.admin().indices().prepareAliases() + .addAlias(INTERNAL_SECURITY_TOKENS_INDEX, SECURITY_TOKENS_ALIAS_NAME) + .execute(ActionListener.wrap(tokensAliasResponse -> { + // now reindex all the other non-token security objects to the new .security-7 index + reindex(parentAwareClient, INTERNAL_SECURITY_INDEX, NEW_INTERNAL_SECURITY_INDEX, nonTokensQuery, + ActionListener.wrap(reindexNonTokensResponse -> { + // move the .security alias to the new index, and remove the old .security-6 index + parentAwareClient.admin().indices().prepareAliases() + .addAlias(NEW_INTERNAL_SECURITY_INDEX, SECURITY_ALIAS_NAME) + .removeAlias(INTERNAL_SECURITY_INDEX, SECURITY_ALIAS_NAME) + .removeIndex(INTERNAL_SECURITY_INDEX) + .execute(ActionListener.wrap(aliasResponse -> { + // merge both reindex responses to return to the action listener + listener.onResponse(new BulkByScrollResponse( + Arrays.asList(reindexTokensResponse, reindexNonTokensResponse), null)); + }, removeReadOnlyConsumer)); + }, removeReadOnlyConsumer)); + }, removeReadOnlyConsumer)); + }, removeReadOnlyConsumer)); + }, listener::onFailure)); + }, listener::onFailure)); + } + + private void createNewSecurityIndices(ParentTaskAssigningClient parentAwareClient, ActionListener listener) { + final Tuple tokensMappingAndSettings = loadMappingAndSettingsSourceFromTemplate(SECURITY_TOKENS_TEMPLATE_NAME); + parentAwareClient.admin().indices().prepareCreate(INTERNAL_SECURITY_TOKENS_INDEX) + .addMapping("doc", tokensMappingAndSettings.v1(), XContentType.JSON) + .setWaitForActiveShards(ActiveShardCount.ALL) + .setSettings(tokensMappingAndSettings.v2()) + .execute(ActionListener.wrap(createTokensIndexResponse -> { + final Tuple objMappingAndSettings = loadMappingAndSettingsSourceFromTemplate(NEW_SECURITY_TEMPLATE_NAME); + parentAwareClient.admin().indices().prepareCreate(NEW_INTERNAL_SECURITY_INDEX) + .addMapping("doc", objMappingAndSettings.v1(), XContentType.JSON) + .setSettings(objMappingAndSettings.v2()) + .setWaitForActiveShards(ActiveShardCount.ALL) + .execute(ActionListener.wrap(createObjIndexResponse -> { + listener.onResponse(new AcknowledgedResponse(true)); + }, listener::onFailure)); + }, listener::onFailure)); + } + + private Tuple loadMappingAndSettingsSourceFromTemplate(String templateName) { + final byte[] template = TemplateUtils + .loadTemplate("/" + templateName + ".json", Version.CURRENT.toString(), TEMPLATE_VERSION_PATTERN) + .getBytes(StandardCharsets.UTF_8); + final PutIndexTemplateRequest request = new PutIndexTemplateRequest(templateName).source(template, XContentType.JSON); + return new Tuple<>(request.mappings().get("doc"), request.settings()); + } + + private void reindex(ParentTaskAssigningClient parentAwareClient, String sourceIndex, String destIndex, QueryBuilder queryBuilder, + ActionListener listener) { + ReindexRequest reindexRequest = new ReindexRequest(); + reindexRequest.setSourceIndices(sourceIndex); + reindexRequest.setSourceDocTypes("doc"); + reindexRequest.setDestIndex(destIndex); + reindexRequest.setSourceQuery(queryBuilder); + reindexRequest.setRefresh(true); + parentAwareClient.execute(ReindexAction.INSTANCE, reindexRequest, listener); + } + + private void removeReadOnlyBlock(ParentTaskAssigningClient parentAwareClient, String index, + ActionListener listener) { + Settings settings = Settings.builder().put(IndexMetaData.INDEX_READ_ONLY_SETTING.getKey(), false).build(); + parentAwareClient.admin().indices().prepareUpdateSettings(index).setSettings(settings).execute(listener); + } + + /** + * Makes the index readonly if it's not set as a readonly yet + */ + private void setReadOnlyBlock(String index, ActionListener listener) { + clusterService.submitStateUpdateTask("lock-index-for-upgrade", new ClusterStateUpdateTask() { + + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + final IndexMetaData indexMetaData = currentState.metaData().index(index); + if (indexMetaData == null) { + throw new IndexNotFoundException(index); + } + + if (indexMetaData.getState() != IndexMetaData.State.OPEN) { + throw new IllegalStateException("unable to upgrade a closed index[" + index + "]"); + } + if (currentState.blocks().hasIndexBlock(index, IndexMetaData.INDEX_READ_ONLY_BLOCK)) { + throw new IllegalStateException("unable to upgrade a read-only index[" + index + "]"); + } + + final Settings indexSettingsBuilder = + Settings.builder() + .put(indexMetaData.getSettings()) + .put(IndexMetaData.INDEX_READ_ONLY_SETTING.getKey(), true) + .build(); + final IndexMetaData.Builder builder = IndexMetaData.builder(indexMetaData).settings(indexSettingsBuilder); + assert same(indexMetaData.getSettings(), indexSettingsBuilder) == false; + builder.settingsVersion(1 + builder.settingsVersion()); + + MetaData.Builder metaDataBuilder = MetaData.builder(currentState.metaData()).put(builder); + + ClusterBlocks.Builder blocks = ClusterBlocks.builder().blocks(currentState.blocks()) + .addIndexBlock(index, IndexMetaData.INDEX_READ_ONLY_BLOCK); + + return ClusterState.builder(currentState).metaData(metaDataBuilder).blocks(blocks).build(); + } + + @Override + public void onFailure(String source, Exception e) { + listener.onFailure(e); + } + + @Override + public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { + listener.onResponse(TransportResponse.Empty.INSTANCE); + } + }); } } From e46ed3130313aebad40a06cf7af16fc048750f05 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Sun, 6 Jan 2019 15:47:35 +0200 Subject: [PATCH 11/19] Plug in SecurityUpgradeCheck --- .../xpack/upgrade/IndexUpgradeCheck.java | 5 ++- .../xpack/upgrade/IndexUpgradeService.java | 8 ++-- .../upgrade/SecurityIndexUpgradeCheck.java | 40 +++++++++++++---- .../elasticsearch/xpack/upgrade/Upgrade.java | 17 ++++++-- .../xpack/upgrade/UpgradeCheck.java | 43 +++++++++++++++++++ 5 files changed, 96 insertions(+), 17 deletions(-) create mode 100644 x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/UpgradeCheck.java diff --git a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/IndexUpgradeCheck.java b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/IndexUpgradeCheck.java index 4537221a407f1..1bc96e66dcdff 100644 --- a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/IndexUpgradeCheck.java +++ b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/IndexUpgradeCheck.java @@ -30,7 +30,7 @@ * - reindex is performed * - postUpgrade is called if reindex was successful */ -public class IndexUpgradeCheck { +public class IndexUpgradeCheck implements UpgradeCheck { private final String name; private final Function actionRequired; @@ -78,6 +78,7 @@ public IndexUpgradeCheck(String name, /** * Returns the name of the check */ + @Override public String getName() { return name; } @@ -88,6 +89,7 @@ public String getName() { * @param indexMetaData index metadata * @return required action or UpgradeActionRequired.NOT_APPLICABLE if this check cannot be performed on the index */ + @Override public UpgradeActionRequired actionRequired(IndexMetaData indexMetaData) { return actionRequired.apply(indexMetaData); } @@ -100,6 +102,7 @@ public UpgradeActionRequired actionRequired(IndexMetaData indexMetaData) { * @param state current cluster state * @param listener the listener that should be called upon completion of the upgrade */ + @Override public void upgrade(TaskId task, IndexMetaData indexMetaData, ClusterState state, ActionListener listener) { reindexer.upgrade(task, indexMetaData.getIndex().getName(), state, listener); diff --git a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/IndexUpgradeService.java b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/IndexUpgradeService.java index fb06c81f25f51..41fb236a3e920 100644 --- a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/IndexUpgradeService.java +++ b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/IndexUpgradeService.java @@ -27,11 +27,11 @@ public class IndexUpgradeService { private static final Logger logger = LogManager.getLogger(IndexUpgradeService.class); - private final List upgradeChecks; + private final List upgradeChecks; private final IndexNameExpressionResolver indexNameExpressionResolver; - public IndexUpgradeService(List upgradeChecks) { + public IndexUpgradeService(List upgradeChecks) { this.upgradeChecks = upgradeChecks; this.indexNameExpressionResolver = new IndexNameExpressionResolver(); } @@ -59,7 +59,7 @@ public Map upgradeInfo(String[] indices, IndicesO } private UpgradeActionRequired upgradeInfo(IndexMetaData indexMetaData, String index) { - for (IndexUpgradeCheck check : upgradeChecks) { + for (UpgradeCheck check : upgradeChecks) { UpgradeActionRequired upgradeActionRequired = check.actionRequired(indexMetaData); logger.trace("[{}] check [{}] returned [{}]", index, check.getName(), upgradeActionRequired); switch (upgradeActionRequired) { @@ -92,7 +92,7 @@ public void upgrade(TaskId task, String index, ClusterState state, ActionListene if (indexMetaData == null) { throw new IndexNotFoundException(index); } - for (IndexUpgradeCheck check : upgradeChecks) { + for (UpgradeCheck check : upgradeChecks) { UpgradeActionRequired upgradeActionRequired = check.actionRequired(indexMetaData); switch (upgradeActionRequired) { case UPGRADE: diff --git a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/SecurityIndexUpgradeCheck.java b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/SecurityIndexUpgradeCheck.java index 0624eac3fb97c..8f878cf124450 100644 --- a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/SecurityIndexUpgradeCheck.java +++ b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/SecurityIndexUpgradeCheck.java @@ -41,7 +41,7 @@ import static org.elasticsearch.index.IndexSettings.same; -public class SecurityIndexUpgradeCheck { +public class SecurityIndexUpgradeCheck implements UpgradeCheck { private static final String SECURITY_ALIAS_NAME = ".security"; private static final String TEMPLATE_VERSION_PATTERN = Pattern.quote("${security.template.version}"); @@ -68,6 +68,7 @@ public SecurityIndexUpgradeCheck(String name, Client client, ClusterService clus /** * Returns the name of the check */ + @Override public String getName() { return name; } @@ -78,11 +79,18 @@ public String getName() { * @param indexMetaData index metadata * @return required action or UpgradeActionRequired.NOT_APPLICABLE if this check cannot be performed on the index */ + @Override public UpgradeActionRequired actionRequired(IndexMetaData indexMetaData) { if (indexMetaData.getIndex().getName().equals(INTERNAL_SECURITY_INDEX)) { - // no need to check the "index.format" setting as the name encodes the version + // no need to check the "index.format" setting value because the name encodes the format version return UpgradeActionRequired.UPGRADE; - } else { + } else if (indexMetaData.getIndex().getName().equals(NEW_INTERNAL_SECURITY_INDEX)) { + // no need to check the "index.format" setting value because the name encodes the format version + return UpgradeActionRequired.UP_TO_DATE; + } else if (indexMetaData.getIndex().getName().equals(INTERNAL_SECURITY_TOKENS_INDEX)) { + // no need to check the "index.format" setting value because the name encodes the format version + return UpgradeActionRequired.UP_TO_DATE; + }else { return UpgradeActionRequired.NOT_APPLICABLE; } } @@ -95,13 +103,21 @@ public UpgradeActionRequired actionRequired(IndexMetaData indexMetaData) { * @param state current cluster state * @param listener the listener that should be called upon completion of the upgrade */ + @Override public void upgrade(TaskId task, IndexMetaData indexMetaData, ClusterState state, ActionListener listener) { final ParentTaskAssigningClient parentAwareClient = new ParentTaskAssigningClient(client, task); final BoolQueryBuilder tokensQuery = QueryBuilders.boolQuery().filter(QueryBuilders.termQuery("doc_type", "token")); final BoolQueryBuilder nonTokensQuery = QueryBuilders.boolQuery() .filter(QueryBuilders.boolQuery().mustNot(QueryBuilders.termQuery("doc_type", "token"))); - final Consumer removeReadOnlyConsumer = e -> removeReadOnlyBlock(parentAwareClient, INTERNAL_SECURITY_INDEX, - ActionListener.wrap(unsetReadOnlyResponse -> listener.onFailure(e), e1 -> listener.onFailure(e))); + final Consumer removeReadOnlyBlock = e -> + removeReadOnlyBlock(parentAwareClient, INTERNAL_SECURITY_INDEX, ActionListener.wrap(unsetReadOnlyResponse -> { + listener.onFailure(e); + }, e1 -> { + e.addSuppressed(e1); + listener.onFailure(e); + })); + // check all nodes can handle the new index format + checkMasterAndDataNodeVersion(state); // create the two new indices, but without their corresponding aliases createNewSecurityIndices(parentAwareClient, ActionListener.wrap(createIndicesResponse -> { // set a read-only block on the original .security-6 index @@ -125,14 +141,20 @@ public void upgrade(TaskId task, IndexMetaData indexMetaData, ClusterState state // merge both reindex responses to return to the action listener listener.onResponse(new BulkByScrollResponse( Arrays.asList(reindexTokensResponse, reindexNonTokensResponse), null)); - }, removeReadOnlyConsumer)); - }, removeReadOnlyConsumer)); - }, removeReadOnlyConsumer)); - }, removeReadOnlyConsumer)); + }, removeReadOnlyBlock)); + }, removeReadOnlyBlock)); + }, removeReadOnlyBlock)); + }, removeReadOnlyBlock)); }, listener::onFailure)); }, listener::onFailure)); } + private void checkMasterAndDataNodeVersion(ClusterState clusterState) { + if (clusterState.nodes().getMinNodeVersion().before(Upgrade.UPGRADE_INTRODUCED)) { + throw new IllegalStateException("All nodes should have at least version [" + Upgrade.UPGRADE_INTRODUCED + "] to upgrade"); + } + } + private void createNewSecurityIndices(ParentTaskAssigningClient parentAwareClient, ActionListener listener) { final Tuple tokensMappingAndSettings = loadMappingAndSettingsSourceFromTemplate(SECURITY_TOKENS_TEMPLATE_NAME); parentAwareClient.admin().indices().prepareCreate(INTERNAL_SECURITY_TOKENS_INDEX) diff --git a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/Upgrade.java b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/Upgrade.java index 9cde2269c5cdd..501bfcca4105b 100644 --- a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/Upgrade.java +++ b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/Upgrade.java @@ -9,6 +9,7 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.client.Client; +import org.elasticsearch.client.OriginSettingClient; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; @@ -42,14 +43,17 @@ import java.util.function.BiFunction; import java.util.function.Supplier; +import static org.elasticsearch.xpack.core.ClientHelper.SECURITY_ORIGIN; + public class Upgrade extends Plugin implements ActionPlugin { public static final Version UPGRADE_INTRODUCED = Version.V_6_7_0; - private final List> upgradeCheckFactories; + private final List> upgradeCheckFactories; public Upgrade() { this.upgradeCheckFactories = new ArrayList<>(); + upgradeCheckFactories.add(getSecurityUpgradeCheckFactory()); } @Override @@ -57,8 +61,8 @@ public Collection createComponents(Client client, ClusterService cluster ResourceWatcherService resourceWatcherService, ScriptService scriptService, NamedXContentRegistry xContentRegistry, Environment environment, NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry) { - List upgradeChecks = new ArrayList<>(upgradeCheckFactories.size()); - for (BiFunction checkFactory : upgradeCheckFactories) { + final List upgradeChecks = new ArrayList<>(upgradeCheckFactories.size()); + for (BiFunction checkFactory : upgradeCheckFactories) { upgradeChecks.add(checkFactory.apply(client, clusterService)); } return Collections.singletonList(new IndexUpgradeService(Collections.unmodifiableList(upgradeChecks))); @@ -83,4 +87,11 @@ public List getRestHandlers(Settings settings, RestController restC ); } + static BiFunction getSecurityUpgradeCheckFactory() { + return (client, clusterService) -> { + final Client clientWithOrigin = new OriginSettingClient(client, SECURITY_ORIGIN); + return new SecurityIndexUpgradeCheck("security", clientWithOrigin, clusterService); + }; + } + } diff --git a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/UpgradeCheck.java b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/UpgradeCheck.java new file mode 100644 index 0000000000000..bf9884fa17e8b --- /dev/null +++ b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/UpgradeCheck.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.upgrade; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.index.reindex.BulkByScrollResponse; +import org.elasticsearch.protocol.xpack.migration.UpgradeActionRequired; +import org.elasticsearch.tasks.TaskId; + +/** + * Interface for all upgrade checks. + */ +public interface UpgradeCheck { + + /** + * Returns the name of the check + */ + public String getName(); + + /** + * This method is called by Upgrade API to verify if upgrade or reindex for this index is required + * + * @param indexMetaData index metadata + * @return required action or UpgradeActionRequired.NOT_APPLICABLE if this check cannot be performed on the index + */ + public UpgradeActionRequired actionRequired(IndexMetaData indexMetaData); + + /** + * Perform the index upgrade + * + * @param task the task that executes the upgrade operation + * @param indexMetaData index metadata + * @param state current cluster state + * @param listener the listener that should be called upon completion of the upgrade + */ + public void upgrade(TaskId task, IndexMetaData indexMetaData, ClusterState state, ActionListener listener); +} From 00c23f71a7d1e65efb70f63f1dfaadf9ce046e3a Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Tue, 8 Jan 2019 14:18:01 +0200 Subject: [PATCH 12/19] ExpiredTokenRemover --- .../core/upgrade/SecurityIndexUpgradeNames.java | 11 +++++++++++ .../org/elasticsearch/xpack/security/Security.java | 3 ++- .../xpack/security/authc/ExpiredTokenRemover.java | 13 +++++++++++-- .../xpack/security/authc/TokenService.java | 10 +++++----- .../security/support/SecurityIndexManager.java | 7 ++++--- 5 files changed, 33 insertions(+), 11 deletions(-) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/upgrade/SecurityIndexUpgradeNames.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/upgrade/SecurityIndexUpgradeNames.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/upgrade/SecurityIndexUpgradeNames.java new file mode 100644 index 0000000000000..65e2e2c1f881e --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/upgrade/SecurityIndexUpgradeNames.java @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.upgrade; + +public class SecurityIndexUpgradeNames { + +} diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index 48346f9278508..1b56f82a7d08e 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -429,9 +429,10 @@ Collection createComponents(Client client, ThreadPool threadPool, Cluste this.auditTrailService.set(auditTrailService); securityIndex.set(SecurityIndexManager.buildSecurityIndexManager(client, clusterService)); + final SecurityIndexManager securityTokensIndex = SecurityIndexManager.buildSecurityTokensIndexManager(client, clusterService); final TokenService tokenService = new TokenService(settings, Clock.systemUTC(), client, securityIndex.get(), - SecurityIndexManager.buildSecurityTokensIndexManager(client, clusterService), clusterService); + securityTokensIndex, clusterService); this.tokenService.set(tokenService); components.add(tokenService); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredTokenRemover.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredTokenRemover.java index 5f0f534163928..41e205c94aab7 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredTokenRemover.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredTokenRemover.java @@ -43,15 +43,24 @@ final class ExpiredTokenRemover extends AbstractRunnable { private final Client client; private final AtomicBoolean inProgress = new AtomicBoolean(false); private final TimeValue timeout; + private final SecurityIndexManager securityIndex; + private final SecurityIndexManager securityTokensIndex; - ExpiredTokenRemover(Settings settings, Client client) { + ExpiredTokenRemover(Settings settings, Client client, SecurityIndexManager securityIndex, SecurityIndexManager securityTokensIndex) { this.client = client; this.timeout = TokenService.DELETE_TIMEOUT.get(settings); + this.securityIndex = securityIndex; + this.securityTokensIndex = securityTokensIndex; } @Override public void doRun() { - DeleteByQueryRequest expiredDbq = new DeleteByQueryRequest(SecurityIndexManager.SECURITY_ALIAS_NAME); + final DeleteByQueryRequest expiredDbq; + if (securityTokensIndex.exists()) { + expiredDbq = new DeleteByQueryRequest(securityTokensIndex.aliasName()); + } else { + expiredDbq = new DeleteByQueryRequest(securityIndex.aliasName()); + } if (timeout != TimeValue.MINUS_ONE) { expiredDbq.setTimeout(timeout); expiredDbq.getSearchRequest().source().timeout(timeout); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java index 8330e0604834b..901405bea9bd5 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java @@ -205,7 +205,7 @@ public TokenService(Settings settings, Clock clock, Client client, SecurityIndex this.lastExpirationRunMs = client.threadPool().relativeTimeInMillis(); this.deleteInterval = DELETE_INTERVAL.get(settings); this.enabled = isTokenServiceEnabled(settings); - this.expiredTokenRemover = new ExpiredTokenRemover(settings, client); + this.expiredTokenRemover = new ExpiredTokenRemover(settings, client, securityIndex, securityTokensIndex); ensureEncryptionCiphersSupported(); KeyAndCache keyAndCache = new KeyAndCache(new KeyAndTimestamp(tokenPassphrase, createdTimeStamps.incrementAndGet()), new BytesKey(saltArr)); @@ -279,10 +279,10 @@ public void createUserToken(Authentication authentication, Authentication origin .setSource(builder) .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL) .request(); - indexManager.prepareIndexIfNeededThenExecute(ex -> listener.onFailure(traceLog("prepare security index", documentId, ex)), - () -> executeAsyncWithOrigin(client, SECURITY_ORIGIN, IndexAction.INSTANCE, request, - ActionListener.wrap(indexResponse -> listener.onResponse(new Tuple<>(userToken, refreshToken)), - listener::onFailure)) + indexManager.prepareIndexIfNeededThenExecute( + ex -> listener.onFailure(traceLog("prepare alias [" + indexManager.aliasName() + "]", documentId, ex)), + () -> executeAsyncWithOrigin(client, SECURITY_ORIGIN, IndexAction.INSTANCE, request, ActionListener + .wrap(indexResponse -> listener.onResponse(new Tuple<>(userToken, refreshToken)), listener::onFailure)) ); } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java index c450da3ef6fe8..95c84da8850e0 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java @@ -122,7 +122,8 @@ public SecurityIndexManager freeze() { } public static List indexNames() { - return Collections.unmodifiableList(Arrays.asList(SECURITY_ALIAS_NAME, INTERNAL_SECURITY_INDEX)); + return Collections.unmodifiableList( + Arrays.asList(SECURITY_ALIAS_NAME, INTERNAL_SECURITY_INDEX, SECURITY_TOKENS_ALIAS_NAME, INTERNAL_SECURITY_TOKENS_INDEX)); } public boolean checkMappingVersion(Predicate requiredVersion) { @@ -155,7 +156,7 @@ public boolean isAvailable() { } /** - * Returns whether the index has the mapping Mapping can change at any version. + * Returns whether the index has the latest mapping. Mapping can change at any version. */ public boolean isMappingUpToDate() { return this.indexState.mappingUpToDate; @@ -196,7 +197,7 @@ public void clusterChanged(ClusterChangedEvent event) { final IndexMetaData indexMetaData = resolveConcreteIndex(aliasName, event.state().metaData()); final boolean indexExists = indexMetaData != null; // if index does not exist it will be created with an up to date version - // if the version is newer that the latest expected, then a manual upgrade has been issued + // if the version is newer than the latest expected, then a manual upgrade has been issued final boolean isIndexUpToDate = indexExists == false || INDEX_FORMAT_SETTING.get(indexMetaData.getSettings()).intValue() >= expectedIndexFormat; // all primary shards allocated From 7dff2162900c5d2f193d9fdde527fa5f4033ec45 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Tue, 8 Jan 2019 14:37:18 +0200 Subject: [PATCH 13/19] Nits --- .../xpack/core/upgrade/SecurityIndexUpgradeNames.java | 11 ----------- .../xpack/upgrade/SecurityIndexUpgradeCheck.java | 4 ++-- 2 files changed, 2 insertions(+), 13 deletions(-) delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/upgrade/SecurityIndexUpgradeNames.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/upgrade/SecurityIndexUpgradeNames.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/upgrade/SecurityIndexUpgradeNames.java deleted file mode 100644 index 65e2e2c1f881e..0000000000000 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/upgrade/SecurityIndexUpgradeNames.java +++ /dev/null @@ -1,11 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.core.upgrade; - -public class SecurityIndexUpgradeNames { - -} diff --git a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/SecurityIndexUpgradeCheck.java b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/SecurityIndexUpgradeCheck.java index 8f878cf124450..35c55aa914e89 100644 --- a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/SecurityIndexUpgradeCheck.java +++ b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/SecurityIndexUpgradeCheck.java @@ -90,7 +90,7 @@ public UpgradeActionRequired actionRequired(IndexMetaData indexMetaData) { } else if (indexMetaData.getIndex().getName().equals(INTERNAL_SECURITY_TOKENS_INDEX)) { // no need to check the "index.format" setting value because the name encodes the format version return UpgradeActionRequired.UP_TO_DATE; - }else { + } else { return UpgradeActionRequired.NOT_APPLICABLE; } } @@ -109,7 +109,7 @@ public void upgrade(TaskId task, IndexMetaData indexMetaData, ClusterState state final BoolQueryBuilder tokensQuery = QueryBuilders.boolQuery().filter(QueryBuilders.termQuery("doc_type", "token")); final BoolQueryBuilder nonTokensQuery = QueryBuilders.boolQuery() .filter(QueryBuilders.boolQuery().mustNot(QueryBuilders.termQuery("doc_type", "token"))); - final Consumer removeReadOnlyBlock = e -> + final Consumer removeReadOnlyBlock = e -> removeReadOnlyBlock(parentAwareClient, INTERNAL_SECURITY_INDEX, ActionListener.wrap(unsetReadOnlyResponse -> { listener.onFailure(e); }, e1 -> { From 4be665e026a8802317f41fd358debca39108f098 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Tue, 8 Jan 2019 15:40:37 +0200 Subject: [PATCH 14/19] add creation time for tokens --- .../src/main/resources/new-security-index-template.json | 8 -------- .../main/resources/security-tokens-index-template.json | 4 ++++ 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/x-pack/plugin/core/src/main/resources/new-security-index-template.json b/x-pack/plugin/core/src/main/resources/new-security-index-template.json index 9e756b0b8b605..dbc540d9cdcd1 100644 --- a/x-pack/plugin/core/src/main/resources/new-security-index-template.json +++ b/x-pack/plugin/core/src/main/resources/new-security-index-template.json @@ -141,14 +141,6 @@ "actions" : { "type" : "keyword" }, - "expiration_time" : { - "type" : "date", - "format" : "epoch_millis" - }, - "creation_time" : { - "type" : "date", - "format" : "epoch_millis" - }, "rules" : { "type" : "object", "dynamic" : true diff --git a/x-pack/plugin/core/src/main/resources/security-tokens-index-template.json b/x-pack/plugin/core/src/main/resources/security-tokens-index-template.json index 88e4d914e5c5e..0cf42f4ea7c53 100644 --- a/x-pack/plugin/core/src/main/resources/security-tokens-index-template.json +++ b/x-pack/plugin/core/src/main/resources/security-tokens-index-template.json @@ -18,6 +18,10 @@ "doc_type" : { "type" : "keyword" }, + "creation_time" : { + "type" : "date", + "format" : "epoch_millis" + }, "refresh_token" : { "type" : "object", "properties" : { From 37489fe0b782b415eaeb40069435762106226b06 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Tue, 8 Jan 2019 18:05:49 +0200 Subject: [PATCH 15/19] Yay!! --- .../xpack/upgrade/SecurityIndexUpgradeCheck.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/SecurityIndexUpgradeCheck.java b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/SecurityIndexUpgradeCheck.java index 35c55aa914e89..a71c038cbebb0 100644 --- a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/SecurityIndexUpgradeCheck.java +++ b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/SecurityIndexUpgradeCheck.java @@ -15,7 +15,6 @@ import org.elasticsearch.client.ParentTaskAssigningClient; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateUpdateTask; -import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.service.ClusterService; @@ -135,7 +134,6 @@ public void upgrade(TaskId task, IndexMetaData indexMetaData, ClusterState state // move the .security alias to the new index, and remove the old .security-6 index parentAwareClient.admin().indices().prepareAliases() .addAlias(NEW_INTERNAL_SECURITY_INDEX, SECURITY_ALIAS_NAME) - .removeAlias(INTERNAL_SECURITY_INDEX, SECURITY_ALIAS_NAME) .removeIndex(INTERNAL_SECURITY_INDEX) .execute(ActionListener.wrap(aliasResponse -> { // merge both reindex responses to return to the action listener @@ -187,6 +185,7 @@ private void reindex(ParentTaskAssigningClient parentAwareClient, String sourceI reindexRequest.setSourceIndices(sourceIndex); reindexRequest.setSourceDocTypes("doc"); reindexRequest.setDestIndex(destIndex); + reindexRequest.setDestDocType("doc"); reindexRequest.setSourceQuery(queryBuilder); reindexRequest.setRefresh(true); parentAwareClient.execute(ReindexAction.INSTANCE, reindexRequest, listener); @@ -229,10 +228,7 @@ public ClusterState execute(ClusterState currentState) throws Exception { MetaData.Builder metaDataBuilder = MetaData.builder(currentState.metaData()).put(builder); - ClusterBlocks.Builder blocks = ClusterBlocks.builder().blocks(currentState.blocks()) - .addIndexBlock(index, IndexMetaData.INDEX_READ_ONLY_BLOCK); - - return ClusterState.builder(currentState).metaData(metaDataBuilder).blocks(blocks).build(); + return ClusterState.builder(currentState).metaData(metaDataBuilder).build(); } @Override From 2d6e1005b6f61b32d5f5a9b585cfc71e39be2d58 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Wed, 9 Jan 2019 14:17:20 +0200 Subject: [PATCH 16/19] Templates have specific index patterns --- .../core/src/main/resources/new-security-index-template.json | 2 +- .../plugin/core/src/main/resources/security-index-template.json | 2 +- .../core/src/main/resources/security-tokens-index-template.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/core/src/main/resources/new-security-index-template.json b/x-pack/plugin/core/src/main/resources/new-security-index-template.json index dbc540d9cdcd1..63f207c5911a4 100644 --- a/x-pack/plugin/core/src/main/resources/new-security-index-template.json +++ b/x-pack/plugin/core/src/main/resources/new-security-index-template.json @@ -1,5 +1,5 @@ { - "index_patterns" : [ ".security-*" ], + "index_patterns" : [ ".security-7" ], "order" : 1000, "settings" : { "number_of_shards" : 1, diff --git a/x-pack/plugin/core/src/main/resources/security-index-template.json b/x-pack/plugin/core/src/main/resources/security-index-template.json index bac5930c0d5c9..aadd85a82db06 100644 --- a/x-pack/plugin/core/src/main/resources/security-index-template.json +++ b/x-pack/plugin/core/src/main/resources/security-index-template.json @@ -1,5 +1,5 @@ { - "index_patterns" : [ ".security-*" ], + "index_patterns" : [ ".security-6" ], "order" : 1000, "settings" : { "number_of_shards" : 1, diff --git a/x-pack/plugin/core/src/main/resources/security-tokens-index-template.json b/x-pack/plugin/core/src/main/resources/security-tokens-index-template.json index 0cf42f4ea7c53..86e99a8b50966 100644 --- a/x-pack/plugin/core/src/main/resources/security-tokens-index-template.json +++ b/x-pack/plugin/core/src/main/resources/security-tokens-index-template.json @@ -1,5 +1,5 @@ { - "index_patterns" : [ ".security-tokens-*" ], + "index_patterns" : [ ".security-tokens-7" ], "order" : 1000, "settings" : { "number_of_shards" : 1, From e70d0d99ca8d127d7f593e66200b51d0e852e774 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Sun, 13 Jan 2019 18:21:19 +0200 Subject: [PATCH 17/19] Renames --- ...ity-index-template.json => security-index-template-6.json} | 0 ...ity-index-template.json => security-index-template-7.json} | 0 ...ex-template.json => security-tokens-index-template-7.json} | 0 .../xpack/security/support/SecurityIndexManager.java | 4 ++-- .../xpack/upgrade/SecurityIndexUpgradeCheck.java | 4 ++-- 5 files changed, 4 insertions(+), 4 deletions(-) rename x-pack/plugin/core/src/main/resources/{security-index-template.json => security-index-template-6.json} (100%) rename x-pack/plugin/core/src/main/resources/{new-security-index-template.json => security-index-template-7.json} (100%) rename x-pack/plugin/core/src/main/resources/{security-tokens-index-template.json => security-tokens-index-template-7.json} (100%) diff --git a/x-pack/plugin/core/src/main/resources/security-index-template.json b/x-pack/plugin/core/src/main/resources/security-index-template-6.json similarity index 100% rename from x-pack/plugin/core/src/main/resources/security-index-template.json rename to x-pack/plugin/core/src/main/resources/security-index-template-6.json diff --git a/x-pack/plugin/core/src/main/resources/new-security-index-template.json b/x-pack/plugin/core/src/main/resources/security-index-template-7.json similarity index 100% rename from x-pack/plugin/core/src/main/resources/new-security-index-template.json rename to x-pack/plugin/core/src/main/resources/security-index-template-7.json diff --git a/x-pack/plugin/core/src/main/resources/security-tokens-index-template.json b/x-pack/plugin/core/src/main/resources/security-tokens-index-template-7.json similarity index 100% rename from x-pack/plugin/core/src/main/resources/security-tokens-index-template.json rename to x-pack/plugin/core/src/main/resources/security-tokens-index-template-7.json diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java index 95c84da8850e0..d783eb35dd54e 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java @@ -69,12 +69,12 @@ public class SecurityIndexManager implements ClusterStateListener { public static final String SECURITY_ALIAS_NAME = ".security"; public static final int INTERNAL_SECURITY_INDEX_FORMAT = 6; public static final String INTERNAL_SECURITY_INDEX = ".security-" + INTERNAL_SECURITY_INDEX_FORMAT; - public static final String SECURITY_TEMPLATE_NAME = "security-index-template"; + public static final String SECURITY_TEMPLATE_NAME = "security-index-template-" + INTERNAL_SECURITY_INDEX_FORMAT; public static final String SECURITY_TOKENS_ALIAS_NAME = ".security-tokens"; public static final int INTERNAL_SECURITY_TOKENS_INDEX_FORMAT = 7; public static final String INTERNAL_SECURITY_TOKENS_INDEX = ".security-tokens-" + INTERNAL_SECURITY_TOKENS_INDEX_FORMAT; - public static final String SECURITY_TOKENS_TEMPLATE_NAME = "security-tokens-index-template"; + public static final String SECURITY_TOKENS_TEMPLATE_NAME = "security-tokens-index-template-" + INTERNAL_SECURITY_TOKENS_INDEX_FORMAT; public static final String SECURITY_VERSION_STRING = "security-version"; public static final String TEMPLATE_VERSION_PATTERN = Pattern.quote("${security.template.version}"); diff --git a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/SecurityIndexUpgradeCheck.java b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/SecurityIndexUpgradeCheck.java index a71c038cbebb0..0618d3242a102 100644 --- a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/SecurityIndexUpgradeCheck.java +++ b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/SecurityIndexUpgradeCheck.java @@ -47,12 +47,12 @@ public class SecurityIndexUpgradeCheck implements UpgradeCheck { private static final int INTERNAL_SECURITY_INDEX_FORMAT = 6; private static final String INTERNAL_SECURITY_INDEX = ".security-" + INTERNAL_SECURITY_INDEX_FORMAT; private static final String NEW_INTERNAL_SECURITY_INDEX = ".security-" + (INTERNAL_SECURITY_INDEX_FORMAT + 1); - private static final String NEW_SECURITY_TEMPLATE_NAME = "new-security-index-template"; + private static final String NEW_SECURITY_TEMPLATE_NAME = "security-index-template-" + (INTERNAL_SECURITY_INDEX_FORMAT + 1); private static final String SECURITY_TOKENS_ALIAS_NAME = ".security-tokens"; private static final int INTERNAL_SECURITY_TOKENS_INDEX_FORMAT = 7; private static final String INTERNAL_SECURITY_TOKENS_INDEX = ".security-tokens-" + INTERNAL_SECURITY_TOKENS_INDEX_FORMAT; - private static final String SECURITY_TOKENS_TEMPLATE_NAME = "security-tokens-index-template"; + private static final String SECURITY_TOKENS_TEMPLATE_NAME = "security-tokens-index-template-" + INTERNAL_SECURITY_TOKENS_INDEX_FORMAT; private final String name; private final Client client; From ff22f81f6252f018adb8593670bb7c8afef001f2 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Sun, 13 Jan 2019 18:37:36 +0200 Subject: [PATCH 18/19] Forgo "invalidated-token" --- .../xpack/upgrade/SecurityIndexUpgradeCheck.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/SecurityIndexUpgradeCheck.java b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/SecurityIndexUpgradeCheck.java index 0618d3242a102..bc4fc653f0b07 100644 --- a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/SecurityIndexUpgradeCheck.java +++ b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/SecurityIndexUpgradeCheck.java @@ -106,8 +106,9 @@ public UpgradeActionRequired actionRequired(IndexMetaData indexMetaData) { public void upgrade(TaskId task, IndexMetaData indexMetaData, ClusterState state, ActionListener listener) { final ParentTaskAssigningClient parentAwareClient = new ParentTaskAssigningClient(client, task); final BoolQueryBuilder tokensQuery = QueryBuilders.boolQuery().filter(QueryBuilders.termQuery("doc_type", "token")); - final BoolQueryBuilder nonTokensQuery = QueryBuilders.boolQuery() - .filter(QueryBuilders.boolQuery().mustNot(QueryBuilders.termQuery("doc_type", "token"))); + // invalidated-token docs are abandoned as they are not used anymore + final BoolQueryBuilder nonTokensQuery = QueryBuilders.boolQuery().filter(QueryBuilders.boolQuery() + .mustNot(QueryBuilders.termQuery("doc_type", "token")).mustNot(QueryBuilders.termQuery("doc_type", "invalidated-token"))); final Consumer removeReadOnlyBlock = e -> removeReadOnlyBlock(parentAwareClient, INTERNAL_SECURITY_INDEX, ActionListener.wrap(unsetReadOnlyResponse -> { listener.onFailure(e); From f030a7f24005d7b4c225bcdb0b37364ca37e20de Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Mon, 14 Jan 2019 01:43:33 +0200 Subject: [PATCH 19/19] StepListener --- .../upgrade/SecurityIndexUpgradeCheck.java | 125 +++++++++++------- 1 file changed, 76 insertions(+), 49 deletions(-) diff --git a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/SecurityIndexUpgradeCheck.java b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/SecurityIndexUpgradeCheck.java index bc4fc653f0b07..66412db127c64 100644 --- a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/SecurityIndexUpgradeCheck.java +++ b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/SecurityIndexUpgradeCheck.java @@ -8,6 +8,7 @@ import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.StepListener; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.master.AcknowledgedResponse; @@ -95,12 +96,25 @@ public UpgradeActionRequired actionRequired(IndexMetaData indexMetaData) { } /** - * Perform the index upgrade + * Perform the security index upgrade. The `.security-6` index is reindexed into `.security-tokens-7` and `.security-7`. The two reindex + * operations are issued sequentially, firstly the reindex from `.security-6` into `.security-tokens-7` of all token documents and then, + * secondly, from `.security-6` into `.security-7` of all non-token documents (roles, users, etc). During this, the `.security-6` is + * marked read-only, and thereafter it is removed. For this reason, tokens cannot be created/invalidated/refreshed (but can be used) in + * the span of time of the first reindex operation. Passwords, roles and role mappings cannot be changed during the whole upgrade + * operation. Normally, token docs are more write intensive compared with the other docs in the `.security-6` index, and this strategy + * aims to minimize the window of time during which certain token operations are unavailable. In case of any unexpected failure in this + * flow no other rectifying action is taken, besides toggling back the read-write flag on `.security-6` - manual intervention is + * required in case of failure, the upgrade cannot be simply rerun. * - * @param task the task that executes the upgrade operation - * @param indexMetaData index metadata - * @param state current cluster state - * @param listener the listener that should be called upon completion of the upgrade + * @param task + * the task that executes the upgrade operation + * @param indexMetaData + * index metadata of the `.security-6` index + * @param state + * current cluster state + * @param listener + * The listener that should be called upon completion of the upgrade. The listener's response will be populated with the + * results of the two reindex operations. */ @Override public void upgrade(TaskId task, IndexMetaData indexMetaData, ClusterState state, ActionListener listener) { @@ -109,52 +123,64 @@ public void upgrade(TaskId task, IndexMetaData indexMetaData, ClusterState state // invalidated-token docs are abandoned as they are not used anymore final BoolQueryBuilder nonTokensQuery = QueryBuilders.boolQuery().filter(QueryBuilders.boolQuery() .mustNot(QueryBuilders.termQuery("doc_type", "token")).mustNot(QueryBuilders.termQuery("doc_type", "invalidated-token"))); - final Consumer removeReadOnlyBlock = e -> - removeReadOnlyBlock(parentAwareClient, INTERNAL_SECURITY_INDEX, ActionListener.wrap(unsetReadOnlyResponse -> { - listener.onFailure(e); - }, e1 -> { - e.addSuppressed(e1); - listener.onFailure(e); - })); + // in case of failure, make sure we don't forget to restore the .security-6 to read-write + final Consumer removeReadOnlyBlock = e -> removeReadOnlyBlock(parentAwareClient, INTERNAL_SECURITY_INDEX, + ActionListener.wrap(unsetReadOnlyResponse -> { + listener.onFailure(e); + }, e1 -> { + e.addSuppressed(e1); + listener.onFailure(e); + })); + // check all nodes can handle the new index format - checkMasterAndDataNodeVersion(state); - // create the two new indices, but without their corresponding aliases - createNewSecurityIndices(parentAwareClient, ActionListener.wrap(createIndicesResponse -> { + if (state.nodes().getMinNodeVersion().before(Upgrade.UPGRADE_INTRODUCED)) { + listener.onFailure( + new IllegalStateException("All nodes should have at least version [" + Upgrade.UPGRADE_INTRODUCED + "] to upgrade")); + } + + // create the two new .security-* indices, but without their corresponding aliases + final StepListener createNewSecurityIndicesStep = createNewSecurityIndices(parentAwareClient); + + final StepListener setReadOnlyStep = new StepListener<>(); + createNewSecurityIndicesStep.whenComplete(createIndicesResponse -> { // set a read-only block on the original .security-6 index - setReadOnlyBlock(INTERNAL_SECURITY_INDEX, ActionListener.wrap(setReadOnlyResponse -> { - // firstly, reindex all the tokens to the new tokens index - reindex(parentAwareClient, INTERNAL_SECURITY_INDEX, INTERNAL_SECURITY_TOKENS_INDEX, tokensQuery, - ActionListener.wrap(reindexTokensResponse -> { - // create the alias pointing to the new tokens index; now the new tokens index is writable anew - parentAwareClient.admin().indices().prepareAliases() - .addAlias(INTERNAL_SECURITY_TOKENS_INDEX, SECURITY_TOKENS_ALIAS_NAME) - .execute(ActionListener.wrap(tokensAliasResponse -> { - // now reindex all the other non-token security objects to the new .security-7 index - reindex(parentAwareClient, INTERNAL_SECURITY_INDEX, NEW_INTERNAL_SECURITY_INDEX, nonTokensQuery, - ActionListener.wrap(reindexNonTokensResponse -> { - // move the .security alias to the new index, and remove the old .security-6 index - parentAwareClient.admin().indices().prepareAliases() - .addAlias(NEW_INTERNAL_SECURITY_INDEX, SECURITY_ALIAS_NAME) - .removeIndex(INTERNAL_SECURITY_INDEX) - .execute(ActionListener.wrap(aliasResponse -> { - // merge both reindex responses to return to the action listener - listener.onResponse(new BulkByScrollResponse( - Arrays.asList(reindexTokensResponse, reindexNonTokensResponse), null)); - }, removeReadOnlyBlock)); - }, removeReadOnlyBlock)); - }, removeReadOnlyBlock)); - }, removeReadOnlyBlock)); - }, listener::onFailure)); - }, listener::onFailure)); - } + setReadOnlyBlock(INTERNAL_SECURITY_INDEX, setReadOnlyStep); + }, listener::onFailure); - private void checkMasterAndDataNodeVersion(ClusterState clusterState) { - if (clusterState.nodes().getMinNodeVersion().before(Upgrade.UPGRADE_INTRODUCED)) { - throw new IllegalStateException("All nodes should have at least version [" + Upgrade.UPGRADE_INTRODUCED + "] to upgrade"); - } + final StepListener reindexTokensStep = new StepListener<>(); + setReadOnlyStep.whenComplete(setReadOnlyResponse -> { + // firstly, reindex all the tokens to the new tokens index + reindex(parentAwareClient, INTERNAL_SECURITY_INDEX, INTERNAL_SECURITY_TOKENS_INDEX, tokensQuery, reindexTokensStep); + }, listener::onFailure); + + final StepListener createTokensAliasStep = new StepListener<>(); + reindexTokensStep.whenComplete(reindexTokensResponse -> { + // then, create the alias .security-tokens pointing to the new tokens index; now the new tokens index/alias is writable anew + parentAwareClient.admin().indices().prepareAliases().addAlias(INTERNAL_SECURITY_TOKENS_INDEX, SECURITY_TOKENS_ALIAS_NAME) + .execute(createTokensAliasStep); + }, removeReadOnlyBlock); + + final StepListener reindexNonTokensStep = new StepListener<>(); + createTokensAliasStep.whenComplete(tokensAliasResponse -> { + // now reindex all the other non-token security objects to the new .security-7 index + reindex(parentAwareClient, INTERNAL_SECURITY_INDEX, NEW_INTERNAL_SECURITY_INDEX, nonTokensQuery, reindexNonTokensStep); + }, removeReadOnlyBlock); + + final StepListener createNonTokensAliasStep = new StepListener<>(); + reindexNonTokensStep.whenComplete(reindexNonTokensResponse -> { + // move the .security alias to the new .security-7 index, and remove the old .security-6 index + parentAwareClient.admin().indices().prepareAliases().addAlias(NEW_INTERNAL_SECURITY_INDEX, SECURITY_ALIAS_NAME) + .removeIndex(INTERNAL_SECURITY_INDEX).execute(createNonTokensAliasStep); + }, removeReadOnlyBlock); + + createNonTokensAliasStep.whenComplete(aliasResponse -> { + // merge both reindex responses to return to the action listener + listener.onResponse(new BulkByScrollResponse(Arrays.asList(reindexTokensStep.result(), reindexNonTokensStep.result()), null)); + }, removeReadOnlyBlock); } - private void createNewSecurityIndices(ParentTaskAssigningClient parentAwareClient, ActionListener listener) { + private StepListener createNewSecurityIndices(ParentTaskAssigningClient parentAwareClient) { + final StepListener createNewSecurityIndicesStep = new StepListener<>(); final Tuple tokensMappingAndSettings = loadMappingAndSettingsSourceFromTemplate(SECURITY_TOKENS_TEMPLATE_NAME); parentAwareClient.admin().indices().prepareCreate(INTERNAL_SECURITY_TOKENS_INDEX) .addMapping("doc", tokensMappingAndSettings.v1(), XContentType.JSON) @@ -167,9 +193,10 @@ private void createNewSecurityIndices(ParentTaskAssigningClient parentAwareClien .setSettings(objMappingAndSettings.v2()) .setWaitForActiveShards(ActiveShardCount.ALL) .execute(ActionListener.wrap(createObjIndexResponse -> { - listener.onResponse(new AcknowledgedResponse(true)); - }, listener::onFailure)); - }, listener::onFailure)); + createNewSecurityIndicesStep.onResponse(new AcknowledgedResponse(true)); + }, createNewSecurityIndicesStep::onFailure)); + }, createNewSecurityIndicesStep::onFailure)); + return createNewSecurityIndicesStep; } private Tuple loadMappingAndSettingsSourceFromTemplate(String templateName) {