diff --git a/runtime/service/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java b/runtime/service/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java index 816fc67986..bfb77ac262 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java @@ -36,6 +36,7 @@ import java.util.Optional; import java.util.Set; import java.util.function.Function; +import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; @@ -51,6 +52,7 @@ import org.apache.polaris.core.admin.model.Catalog; import org.apache.polaris.core.admin.model.CatalogGrant; import org.apache.polaris.core.admin.model.CatalogPrivilege; +import org.apache.polaris.core.admin.model.CatalogRole; import org.apache.polaris.core.admin.model.ConnectionConfigInfo; import org.apache.polaris.core.admin.model.CreateCatalogRequest; import org.apache.polaris.core.admin.model.ExternalCatalog; @@ -60,6 +62,8 @@ import org.apache.polaris.core.admin.model.OAuthClientCredentialsParameters; import org.apache.polaris.core.admin.model.PolicyGrant; import org.apache.polaris.core.admin.model.PolicyPrivilege; +import org.apache.polaris.core.admin.model.Principal; +import org.apache.polaris.core.admin.model.PrincipalRole; import org.apache.polaris.core.admin.model.PrincipalWithCredentials; import org.apache.polaris.core.admin.model.PrincipalWithCredentialsCredentials; import org.apache.polaris.core.admin.model.TableGrant; @@ -82,6 +86,7 @@ import org.apache.polaris.core.entity.NamespaceEntity; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisEntity; +import org.apache.polaris.core.entity.PolarisEntityCore; import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PolarisGrantRecord; @@ -200,6 +205,43 @@ private Optional findCatalogRoleByName(String catalogName, St .map(path -> CatalogRoleEntity.of(path.getRawLeafEntity())); } + private Stream loadEntities( + @Nonnull PolarisEntityType entityType, + @Nonnull PolarisEntitySubType entitySubType, + @Nullable PolarisEntity catalogEntity, + @Nonnull Function transformer) { + List catalogPath; + long catalogId; + if (catalogEntity == null) { + catalogPath = null; + catalogId = 0; + } else { + catalogPath = PolarisEntity.toCoreList(List.of(catalogEntity)); + catalogId = catalogEntity.getId(); + } + // TODO: add loadEntities method to PolarisMetaStoreManager + // loadEntity may return null due to multiple non-atomic API calls to the persistence layer. + // Specifically, this can happen when a PolarisEntity is returned by listEntities, but cannot be + // loaded afterward because it was purged by another process before it could be loaded. + return metaStoreManager + .listEntities( + getCurrentPolarisContext(), + catalogPath, + entityType, + entitySubType, + PageToken.readEverything()) + .getEntities() + .stream() + .map( + nameAndId -> + metaStoreManager.loadEntity( + getCurrentPolarisContext(), catalogId, nameAndId.getId(), nameAndId.getType())) + .map(PolarisEntity::of) + .filter(Objects::nonNull) + .map(transformer) + .filter(Objects::nonNull); + } + private void authorizeBasicRootOperationOrThrow(PolarisAuthorizableOperation op) { resolutionManifest = resolutionManifestFactory.createResolutionManifest( @@ -618,8 +660,7 @@ private boolean catalogOverlapsWithExistingCatalog(CatalogEntity catalogEntity) } Set newCatalogLocations = getCatalogLocations(catalogEntity); - return listCatalogsUnsafe().stream() - .map(CatalogEntity::new) + return listCatalogsUnsafe() .anyMatch( existingCatalog -> { if (existingCatalog.getName().equals(catalogEntity.getName())) { @@ -944,40 +985,16 @@ private void validateUpdateCatalogDiffOrThrow( return returnedEntity; } - /** - * List all catalogs after checking for permission. Nulls due to non-atomic list-then-get are - * filtered out. - */ - public List listCatalogs() { + /** List all catalogs after checking for permission. */ + public List listCatalogs() { authorizeBasicRootOperationOrThrow(PolarisAuthorizableOperation.LIST_CATALOGS); - return listCatalogsUnsafe(); + return listCatalogsUnsafe().map(CatalogEntity::asCatalog).toList(); } - /** - * List all catalogs without checking for permission. Nulls due to non-atomic list-then-get are - * filtered out. - */ - private List listCatalogsUnsafe() { - // loadEntity may return null due to multiple non-atomic - // API calls to the persistence layer. Specifically, this can happen when a PolarisEntity is - // returned by listCatalogs, but cannot be loaded afterward because it was purged by another - // process before it could be loaded. - return metaStoreManager - .listEntities( - getCurrentPolarisContext(), - null, - PolarisEntityType.CATALOG, - PolarisEntitySubType.ANY_SUBTYPE, - PageToken.readEverything()) - .getEntities() - .stream() - .map( - nameAndId -> - PolarisEntity.of( - metaStoreManager.loadEntity( - getCurrentPolarisContext(), 0, nameAndId.getId(), nameAndId.getType()))) - .filter(Objects::nonNull) - .toList(); + /** List all catalogs without checking for permission. */ + private Stream listCatalogsUnsafe() { + return loadEntities( + PolarisEntityType.CATALOG, PolarisEntitySubType.ANY_SUBTYPE, null, CatalogEntity::of); } public PrincipalWithCredentials createPrincipal(PolarisEntity entity) { @@ -1141,24 +1158,16 @@ public void deletePrincipal(String name) { return rotateOrResetCredentialsHelper(principalName, true); } - public List listPrincipals() { + public List listPrincipals() { PolarisAuthorizableOperation op = PolarisAuthorizableOperation.LIST_PRINCIPALS; authorizeBasicRootOperationOrThrow(op); - return metaStoreManager - .listEntities( - getCurrentPolarisContext(), - null, + return loadEntities( PolarisEntityType.PRINCIPAL, PolarisEntitySubType.NULL_SUBTYPE, - PageToken.readEverything()) - .getEntities() - .stream() - .map( - nameAndId -> - PolarisEntity.of( - metaStoreManager.loadEntity( - getCurrentPolarisContext(), 0, nameAndId.getId(), nameAndId.getType()))) + null, + PrincipalEntity::of) + .map(PrincipalEntity::asPrincipal) .toList(); } @@ -1254,24 +1263,16 @@ public void deletePrincipalRole(String name) { return returnedEntity; } - public List listPrincipalRoles() { + public List listPrincipalRoles() { PolarisAuthorizableOperation op = PolarisAuthorizableOperation.LIST_PRINCIPAL_ROLES; authorizeBasicRootOperationOrThrow(op); - return metaStoreManager - .listEntities( - getCurrentPolarisContext(), - null, + return loadEntities( PolarisEntityType.PRINCIPAL_ROLE, PolarisEntitySubType.NULL_SUBTYPE, - PageToken.readEverything()) - .getEntities() - .stream() - .map( - nameAndId -> - PolarisEntity.of( - metaStoreManager.loadEntity( - getCurrentPolarisContext(), 0, nameAndId.getId(), nameAndId.getType()))) + null, + PrincipalRoleEntity::of) + .map(PrincipalRoleEntity::asPrincipalRole) .toList(); } @@ -1383,30 +1384,19 @@ public void deleteCatalogRole(String catalogName, String name) { return returnedEntity; } - public List listCatalogRoles(String catalogName) { + public List listCatalogRoles(String catalogName) { PolarisAuthorizableOperation op = PolarisAuthorizableOperation.LIST_CATALOG_ROLES; authorizeBasicTopLevelEntityOperationOrThrow(op, catalogName, PolarisEntityType.CATALOG); PolarisEntity catalogEntity = findCatalogByName(catalogName) .orElseThrow(() -> new NotFoundException("Parent catalog %s not found", catalogName)); - return metaStoreManager - .listEntities( - getCurrentPolarisContext(), - PolarisEntity.toCoreList(List.of(catalogEntity)), + return loadEntities( PolarisEntityType.CATALOG_ROLE, PolarisEntitySubType.NULL_SUBTYPE, - PageToken.readEverything()) - .getEntities() - .stream() - .map( - nameAndId -> - PolarisEntity.of( - metaStoreManager.loadEntity( - getCurrentPolarisContext(), - catalogEntity.getId(), - nameAndId.getId(), - nameAndId.getType()))) + catalogEntity, + CatalogRoleEntity::of) + .map(CatalogRoleEntity::asCatalogRole) .toList(); } diff --git a/runtime/service/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java b/runtime/service/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java index 8670b1aebe..5318c00b8f 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java @@ -239,11 +239,7 @@ public Response updateCatalog( @Override public Response listCatalogs(RealmContext realmContext, SecurityContext securityContext) { PolarisAdminService adminService = newAdminService(realmContext, securityContext); - List catalogList = - adminService.listCatalogs().stream() - .map(CatalogEntity::new) - .map(CatalogEntity::asCatalog) - .toList(); + List catalogList = adminService.listCatalogs(); Catalogs catalogs = new Catalogs(catalogList); LOGGER.debug("listCatalogs returning: {}", catalogs); return Response.ok(catalogs).build(); @@ -311,11 +307,7 @@ public Response rotateCredentials( @Override public Response listPrincipals(RealmContext realmContext, SecurityContext securityContext) { PolarisAdminService adminService = newAdminService(realmContext, securityContext); - List principalList = - adminService.listPrincipals().stream() - .map(PrincipalEntity::new) - .map(PrincipalEntity::asPrincipal) - .toList(); + List principalList = adminService.listPrincipals(); Principals principals = new Principals(principalList); LOGGER.debug("listPrincipals returning: {}", principals); return Response.ok(principals).build(); @@ -375,11 +367,7 @@ public Response updatePrincipalRole( @Override public Response listPrincipalRoles(RealmContext realmContext, SecurityContext securityContext) { PolarisAdminService adminService = newAdminService(realmContext, securityContext); - List principalRoleList = - adminService.listPrincipalRoles().stream() - .map(PrincipalRoleEntity::new) - .map(PrincipalRoleEntity::asPrincipalRole) - .toList(); + List principalRoleList = adminService.listPrincipalRoles(); PrincipalRoles principalRoles = new PrincipalRoles(principalRoleList); LOGGER.debug("listPrincipalRoles returning: {}", principalRoles); return Response.ok(principalRoles).build(); @@ -451,11 +439,7 @@ public Response updateCatalogRole( public Response listCatalogRoles( String catalogName, RealmContext realmContext, SecurityContext securityContext) { PolarisAdminService adminService = newAdminService(realmContext, securityContext); - List catalogRoleList = - adminService.listCatalogRoles(catalogName).stream() - .map(CatalogRoleEntity::new) - .map(CatalogRoleEntity::asCatalogRole) - .toList(); + List catalogRoleList = adminService.listCatalogRoles(catalogName); CatalogRoles catalogRoles = new CatalogRoles(catalogRoleList); LOGGER.debug("listCatalogRoles returning: {}", catalogRoles); return Response.ok(catalogRoles).build(); diff --git a/runtime/service/src/test/java/org/apache/polaris/service/admin/ManagementServiceTest.java b/runtime/service/src/test/java/org/apache/polaris/service/admin/ManagementServiceTest.java index ab8a58090b..be0067e6af 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/admin/ManagementServiceTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/admin/ManagementServiceTest.java @@ -42,7 +42,6 @@ import org.apache.polaris.core.auth.PolarisAuthorizerImpl; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.PolarisBaseEntity; -import org.apache.polaris.core.entity.PolarisEntity; import org.apache.polaris.core.entity.PolarisEntityConstants; import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; @@ -310,8 +309,8 @@ public void testCatalogNotReturnedWhenDeletedAfterListBeforeGet() { .when(metaStoreManager) .loadEntity(Mockito.any(), Mockito.anyLong(), Mockito.anyLong(), Mockito.any()); - List catalogs = polarisAdminService.listCatalogs(); + List catalogs = polarisAdminService.listCatalogs(); assertThat(catalogs.size()).isEqualTo(1); - assertThat(catalogs.getFirst().getId()).isEqualTo(catalog2.getCatalog().getId()); + assertThat(catalogs.getFirst().getName()).isEqualTo(catalog2.getCatalog().getName()); } }