merge from main, address comments from @flyrain

Author: adnanhemani

CHANGELOG.md

@@ -88,6 +88,8 @@ automatic storage credential refresh per table on the client side. Java client v
 The endpoint path is always returned when using vended credentials, but clients must enable the 
 refresh-credentials flag for the desired storage provider.
 
+- Added a Management API endpoint to reset principal credentials, controlled by the `ENABLE_CREDENTIAL_RESET` (default: true) feature flag.
+
 ### Changes
 
 - Polaris Management API clients must be prepared to deal with new attributes in `AwsStorageConfigInfo` objects.

gradle/baselibs.versions.toml

@@ -23,5 +23,5 @@ idea-ext = { module = "gradle.plugin.org.jetbrains.gradle.plugin.idea-ext:gradle
 jandex = { module = "org.kordamp.gradle:jandex-gradle-plugin", version = "2.2.0" }
 license-report = { module = "com.github.jk1:gradle-license-report", version = "2.9" }
 nexus-publish = { module = "io.github.gradle-nexus:publish-plugin", version = "2.0.0" }
-shadow = { module = "com.gradleup.shadow:shadow-gradle-plugin", version = "9.0.2" }
+shadow = { module = "com.gradleup.shadow:shadow-gradle-plugin", version = "9.1.0" }
 spotless = { module = "com.diffplug.spotless:spotless-plugin-gradle", version = "7.2.1" }

gradle/libs.versions.toml

@@ -41,7 +41,7 @@ swagger = "1.6.16"
 antlr4-runtime = { module = "org.antlr:antlr4-runtime", version.strictly = "4.9.3" } # spark integration tests
 assertj-core = { module = "org.assertj:assertj-core", version = "3.27.4" }
 auth0-jwt = { module = "com.auth0:java-jwt", version = "4.5.0" }
-awssdk-bom = { module = "software.amazon.awssdk:bom", version = "2.32.29" }
+awssdk-bom = { module = "software.amazon.awssdk:bom", version = "2.33.0" }
 awaitility = { module = "org.awaitility:awaitility", version = "4.3.0" }
 azuresdk-bom = { module = "com.azure:azure-sdk-bom", version = "1.2.37" }
 caffeine = { module = "com.github.ben-manes.caffeine:caffeine", version = "3.2.2" }
@@ -90,7 +90,7 @@ prometheus-metrics-exporter-servlet-jakarta = { module = "io.prometheus:promethe
 quarkus-bom = { module = "io.quarkus.platform:quarkus-bom", version.ref = "quarkus" }
 scala212-lang-library = { module = "org.scala-lang:scala-library", version.ref = "scala212" }
 scala212-lang-reflect = { module = "org.scala-lang:scala-reflect", version.ref = "scala212" }
-s3mock-testcontainers = { module = "com.adobe.testing:s3mock-testcontainers", version = "4.7.0" }
+s3mock-testcontainers = { module = "com.adobe.testing:s3mock-testcontainers", version = "4.8.0" }
 slf4j-api = { module = "org.slf4j:slf4j-api", version.ref = "slf4j" }
 smallrye-common-annotation = { module = "io.smallrye.common:smallrye-common-annotation", version = "2.13.8" }
 smallrye-config-core = { module = "io.smallrye.config:smallrye-config-core", version = "3.13.4" }

integration-tests/src/main/java/org/apache/polaris/service/it/env/ManagementApi.java

@@ -80,6 +80,20 @@ public PrincipalWithCredentials createPrincipal(CreatePrincipalRequest request)
     }
   }
 
+  /**
+   * Retrieves a Principal by name via the management API.
+   *
+   * @param principalName the name of the principal to fetch
+   * @return the Principal object
+   */
+  public Principal getPrincipal(String principalName) {
+    try (Response response =
+        request("v1/principals/{principalName}", Map.of("principalName", principalName)).get()) {
+      assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus);
+      return response.readEntity(Principal.class);
+    }
+  }
+
   public void createPrincipalRole(String name) {
     createPrincipalRole(new PrincipalRole(name));
   }

integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisManagementServiceIntegrationTest.java

@@ -880,6 +880,57 @@ public void testCreatePrincipalAndRotateCredentials() {
     // rotation that makes the old secret fall off retention.
   }
 
+  @Test
+  public void testCreatePrincipalAndResetCredentialsWithCustomValues() {
+    //  Create a new principal using root user
+    Principal principal =
+        Principal.builder()
+            .setName(client.newEntityName("myprincipal-reset"))
+            .setProperties(Map.of("custom-tag", "bar"))
+            .build();
+
+    PrincipalWithCredentials creds =
+        managementApi.createPrincipal(new CreatePrincipalRequest(principal, true));
+
+    Map<String, String> customBody =
+        Map.of(
+            "clientId", "f174b76a7e1a99e2",
+            "clientSecret", "27029d236abc08e204922b0a07031bc2");
+
+    PrincipalWithCredentials resetCreds;
+    try (Response response =
+        managementApi
+            .request("v1/principals/{p}/reset", Map.of("p", principal.getName()))
+            .post(Entity.json(customBody))) {
+
+      assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus);
+      resetCreds = response.readEntity(PrincipalWithCredentials.class);
+    }
+
+    assertThat(resetCreds.getCredentials().getClientId()).isEqualTo("f174b76a7e1a99e2");
+    assertThat(resetCreds.getCredentials().getClientSecret())
+        .isEqualTo("27029d236abc08e204922b0a07031bc2");
+
+    // Validate that the principal entity itself is updated in sync with credentials
+    Principal updatedPrincipal = managementApi.getPrincipal(principal.getName());
+    assertThat(updatedPrincipal.getClientId()).isEqualTo("f174b76a7e1a99e2");
+
+    // Principal itself tries to reset with custom creds → should fail (403 Forbidden)
+    String principalToken = client.obtainToken(resetCreds);
+    customBody =
+        Map.of(
+            "clientId", "a174b76a7e1a99e3",
+            "clientSecret", "37029d236abc08e204922b0a07031bc3");
+    try (Response response =
+        client
+            .managementApi(principalToken)
+            .request("v1/principals/{p}/reset", Map.of("p", principal.getName()))
+            .post(Entity.json(customBody))) {
+
+      assertThat(response).returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus);
+    }
+  }
+
   @Test
   public void testCreateFederatedPrincipalRoleSucceeds() {
     // Create a federated Principal Role

integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java

@@ -20,7 +20,6 @@
 
 import static org.apache.polaris.service.it.env.PolarisClient.polarisClient;
 
-import com.google.common.collect.ImmutableMap;
 import java.lang.reflect.Method;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -88,6 +87,7 @@ public abstract class PolarisRestCatalogViewIntegrationBase extends ViewCatalogT
   private static ManagementApi managementApi;
 
   private RESTCatalog restCatalog;
+  private StorageConfigInfo storageConfig;
 
   @BeforeAll
   static void setup(PolarisApiEndpoints apiEndpoints, ClientCredentials credentials) {
@@ -114,7 +114,7 @@ public void before(TestInfo testInfo) {
     Method method = testInfo.getTestMethod().orElseThrow();
     String catalogName = client.newEntityName(method.getName());
 
-    StorageConfigInfo storageConfig = getStorageConfigInfo();
+    storageConfig = getStorageConfigInfo();
     String defaultBaseLocation =
         storageConfig.getAllowedLocations().getFirst()
             + "/"
@@ -201,16 +201,10 @@ public void createViewWithCustomMetadataLocationUsingPolaris(@TempDir Path tempD
     TableIdentifier identifier = TableIdentifier.of("ns", "view");
 
     String location = Paths.get(tempDir.toUri().toString()).toString();
-    String customLocation = Paths.get(tempDir.toUri().toString(), "custom-location").toString();
-    String customLocation2 = Paths.get(tempDir.toUri().toString(), "custom-location2").toString();
-    String customLocationChild =
-        Paths.get(tempDir.toUri().toString(), "custom-location/child").toString();
+    String customLocation =
+        Paths.get(storageConfig.getAllowedLocations().getFirst(), "/custom-location1").toString();
 
-    catalog()
-        .createNamespace(
-            identifier.namespace(),
-            ImmutableMap.of(
-                IcebergTableLikeEntity.USER_SPECIFIED_WRITE_METADATA_LOCATION_KEY, location));
+    catalog().createNamespace(identifier.namespace());
 
     Assertions.assertThat(catalog().viewExists(identifier)).as("View should not exist").isFalse();
 
@@ -234,35 +228,5 @@ public void createViewWithCustomMetadataLocationUsingPolaris(@TempDir Path tempD
     Assertions.assertThat(((BaseView) view).operations().current().metadataFileLocation())
         .isNotNull()
         .startsWith(customLocation);
-
-    // CANNOT update the view with a new metadata location `baseLocation/customLocation2`,
-    // even though the new location is still under the parent namespace's
-    // `write.metadata.path=baseLocation`.
-    Assertions.assertThatThrownBy(
-            () ->
-                catalog()
-                    .loadView(identifier)
-                    .updateProperties()
-                    .set(
-                        IcebergTableLikeEntity.USER_SPECIFIED_WRITE_METADATA_LOCATION_KEY,
-                        customLocation2)
-                    .commit())
-        .isInstanceOf(ForbiddenException.class)
-        .hasMessageContaining("Forbidden: Invalid locations");
-
-    // CANNOT update the view with a child metadata location `baseLocation/customLocation/child`,
-    // even though it is a subpath of the original view's
-    // `write.metadata.path=baseLocation/customLocation`.
-    Assertions.assertThatThrownBy(
-            () ->
-                catalog()
-                    .loadView(identifier)
-                    .updateProperties()
-                    .set(
-                        IcebergTableLikeEntity.USER_SPECIFIED_WRITE_METADATA_LOCATION_KEY,
-                        customLocationChild)
-                    .commit())
-        .isInstanceOf(ForbiddenException.class)
-        .hasMessageContaining("Forbidden: Invalid locations");
   }
 }

persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java

@@ -747,4 +747,15 @@ Optional<Optional<String>> hasOverlappingSiblings(
           @Nonnull PolarisCallContext callContext, T entity) {
     return Optional.empty();
   }
+
+  @Nullable
+  @Override
+  public PolarisPrincipalSecrets storePrincipalSecrets(
+      @Nonnull PolarisCallContext callCtx,
+      long principalId,
+      @Nonnull String resolvedClientId,
+      String customClientSecret) {
+    throw new UnsupportedOperationException(
+        "This method is not supported for EclipseLink as metastore");
+  }
 }

persistence/eclipselink/src/test/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreManagerTest.java

@@ -89,7 +89,7 @@ protected PolarisTestMetaStoreManager createPolarisTestMetaStoreManager() {
             diagServices, store, Mockito.mock(), realmContext, null, "polaris", RANDOM_SECRETS);
     TransactionalMetaStoreManagerImpl metaStoreManager =
         new TransactionalMetaStoreManagerImpl(clock, diagServices);
-    PolarisCallContext callCtx = new PolarisCallContext(realmContext, session, diagServices);
+    PolarisCallContext callCtx = new PolarisCallContext(realmContext, session);
     return new PolarisTestMetaStoreManager(metaStoreManager, callCtx);
   }
 

persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBasePersistenceImpl.java

@@ -37,6 +37,7 @@
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
 import org.apache.polaris.core.PolarisCallContext;
+import org.apache.polaris.core.PolarisDiagnostics;
 import org.apache.polaris.core.entity.EntityNameLookupRecord;
 import org.apache.polaris.core.entity.LocationBasedEntity;
 import org.apache.polaris.core.entity.PolarisBaseEntity;
@@ -79,6 +80,7 @@ public class JdbcBasePersistenceImpl implements BasePersistence, IntegrationPers
 
   private static final Logger LOGGER = LoggerFactory.getLogger(JdbcBasePersistenceImpl.class);
 
+  private final PolarisDiagnostics diagnostics;
   private final DatasourceOperations datasourceOperations;
   private final PrincipalSecretsGenerator secretsGenerator;
   private final PolarisStorageIntegrationProvider storageIntegrationProvider;
@@ -89,11 +91,13 @@ public class JdbcBasePersistenceImpl implements BasePersistence, IntegrationPers
   private static final int MAX_LOCATION_COMPONENTS = 40;
 
   public JdbcBasePersistenceImpl(
+      PolarisDiagnostics diagnostics,
       DatasourceOperations databaseOperations,
       PrincipalSecretsGenerator secretsGenerator,
       PolarisStorageIntegrationProvider storageIntegrationProvider,
       String realmId,
       int schemaVersion) {
+    this.diagnostics = diagnostics;
     this.datasourceOperations = databaseOperations;
     this.secretsGenerator = secretsGenerator;
     this.storageIntegrationProvider = storageIntegrationProvider;
@@ -832,6 +836,42 @@ public PolarisPrincipalSecrets generateNewPrincipalSecrets(
     return principalSecrets;
   }
 
+  @Nullable
+  @Override
+  public PolarisPrincipalSecrets storePrincipalSecrets(
+      @Nonnull PolarisCallContext callCtx,
+      long principalId,
+      @Nonnull String resolvedClientId,
+      String customClientSecret) {
+    PolarisPrincipalSecrets principalSecrets =
+        new PolarisPrincipalSecrets(principalId, resolvedClientId, customClientSecret);
+    try {
+      ModelPrincipalAuthenticationData modelPrincipalAuthenticationData =
+          ModelPrincipalAuthenticationData.fromPrincipalAuthenticationData(principalSecrets);
+      datasourceOperations.executeUpdate(
+          QueryGenerator.generateInsertQuery(
+              ModelPrincipalAuthenticationData.ALL_COLUMNS,
+              ModelPrincipalAuthenticationData.TABLE_NAME,
+              modelPrincipalAuthenticationData
+                  .toMap(datasourceOperations.getDatabaseType())
+                  .values()
+                  .stream()
+                  .toList(),
+              realmId));
+    } catch (SQLException e) {
+      LOGGER.error(
+          "Failed to reset PrincipalSecrets  for clientId: {}, due to {}",
+          resolvedClientId,
+          e.getMessage(),
+          e);
+      throw new RuntimeException(
+          String.format("Failed to reset PrincipalSecrets for clientId: %s", resolvedClientId), e);
+    }
+
+    // return those
+    return principalSecrets;
+  }
+
   @Nullable
   @Override
   public PolarisPrincipalSecrets rotatePrincipalSecrets(
@@ -844,24 +884,20 @@ public PolarisPrincipalSecrets rotatePrincipalSecrets(
     PolarisPrincipalSecrets principalSecrets = loadPrincipalSecrets(callCtx, clientId);
 
     // should be found
-    callCtx
-        .getDiagServices()
-        .checkNotNull(
-            principalSecrets,
-            "cannot_find_secrets",
-            "client_id={} principalId={}",
-            clientId,
-            principalId);
+    diagnostics.checkNotNull(
+        principalSecrets,
+        "cannot_find_secrets",
+        "client_id={} principalId={}",
+        clientId,
+        principalId);
 
     // ensure principal id is matching
-    callCtx
-        .getDiagServices()
-        .check(
-            principalId == principalSecrets.getPrincipalId(),
-            "principal_id_mismatch",
-            "expectedId={} id={}",
-            principalId,
-            principalSecrets.getPrincipalId());
+    diagnostics.check(
+        principalId == principalSecrets.getPrincipalId(),
+        "principal_id_mismatch",
+        "expectedId={} id={}",
+        principalId,
+        principalSecrets.getPrincipalId());
 
     // rotate the secrets
     principalSecrets.rotateSecrets(oldSecretHash);
@@ -1178,7 +1214,7 @@ public <T extends PolarisStorageConfigurationInfo> void persistStorageIntegratio
       PolarisStorageIntegration<T> loadPolarisStorageIntegration(
           @Nonnull PolarisCallContext callContext, @Nonnull PolarisBaseEntity entity) {
     PolarisStorageConfigurationInfo storageConfig =
-        BaseMetaStoreManager.extractStorageConfiguration(callContext.getDiagServices(), entity);
+        BaseMetaStoreManager.extractStorageConfiguration(diagnostics, entity);
     return storageIntegrationProvider.getStorageIntegrationForConfig(storageConfig);
   }
 

persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java

@@ -103,6 +103,7 @@ private void initializeForRealm(
         realmId,
         () ->
             new JdbcBasePersistenceImpl(
+                diagnostics,
                 datasourceOperations,
                 secretsGenerator(realmId, rootCredentialsSet),
                 storageIntegrationProvider,
@@ -177,7 +178,7 @@ public Map<String, BaseResult> purgeRealms(Iterable<String> realms) {
       PolarisMetaStoreManager metaStoreManager = getOrCreateMetaStoreManager(realmContext);
       BasePersistence session = getOrCreateSession(realmContext);
 
-      PolarisCallContext callContext = new PolarisCallContext(realmContext, session, diagnostics);
+      PolarisCallContext callContext = new PolarisCallContext(realmContext, session);
       BaseResult result = metaStoreManager.purge(callContext);
       results.put(realm, result);
 
@@ -216,7 +217,7 @@ public synchronized EntityCache getOrCreateEntityCache(
       PolarisMetaStoreManager metaStoreManager = getOrCreateMetaStoreManager(realmContext);
       entityCacheMap.put(
           realmContext.getRealmIdentifier(),
-          new InMemoryEntityCache(realmConfig, metaStoreManager));
+          new InMemoryEntityCache(diagnostics, realmConfig, metaStoreManager));
     }
 
     return entityCacheMap.get(realmContext.getRealmIdentifier());
@@ -233,8 +234,7 @@ private PrincipalSecretsResult bootstrapServiceAndCreatePolarisPrincipalForRealm
     PolarisMetaStoreManager metaStoreManager =
         metaStoreManagerMap.get(realmContext.getRealmIdentifier());
     BasePersistence metaStore = sessionSupplierMap.get(realmContext.getRealmIdentifier()).get();
-    PolarisCallContext polarisContext =
-        new PolarisCallContext(realmContext, metaStore, diagnostics);
+    PolarisCallContext polarisContext = new PolarisCallContext(realmContext, metaStore);
 
     Optional<PrincipalEntity> preliminaryRootPrincipal =
         metaStoreManager.findRootPrincipal(polarisContext);
@@ -268,8 +268,7 @@ private void checkPolarisServiceBootstrappedForRealm(RealmContext realmContext)
     PolarisMetaStoreManager metaStoreManager =
         metaStoreManagerMap.get(realmContext.getRealmIdentifier());
     BasePersistence metaStore = sessionSupplierMap.get(realmContext.getRealmIdentifier()).get();
-    PolarisCallContext polarisContext =
-        new PolarisCallContext(realmContext, metaStore, diagnostics);
+    PolarisCallContext polarisContext = new PolarisCallContext(realmContext, metaStore);
 
     Optional<PrincipalEntity> rootPrincipal = metaStoreManager.findRootPrincipal(polarisContext);
     if (rootPrincipal.isEmpty()) {