From 34a8db596d33763af1dccc5a6a02a868d5221c6d Mon Sep 17 00:00:00 2001 From: andrew4699 Date: Mon, 11 Nov 2024 09:59:48 -0800 Subject: [PATCH 1/6] Add DynamicFeatureConfigResolver --- .../config/DefaultConfigurationStore.java | 14 +++++++-- .../config/DynamicFeatureConfigResolver.java | 26 +++++++++++++++++ .../NoOpDynamicFeatureConfigResolver.java | 29 +++++++++++++++++++ .../config/PolarisApplicationConfig.java | 15 +++++++++- ...ervice.config.DynamicFeatureConfigResolver | 20 +++++++++++++ .../config/DefaultConfigurationStoreTest.java | 3 +- 6 files changed, 103 insertions(+), 4 deletions(-) create mode 100644 polaris-service/src/main/java/org/apache/polaris/service/config/DynamicFeatureConfigResolver.java create mode 100644 polaris-service/src/main/java/org/apache/polaris/service/config/NoOpDynamicFeatureConfigResolver.java create mode 100644 polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.config.DynamicFeatureConfigResolver diff --git a/polaris-service/src/main/java/org/apache/polaris/service/config/DefaultConfigurationStore.java b/polaris-service/src/main/java/org/apache/polaris/service/config/DefaultConfigurationStore.java index 7f04bbd50c..c84e3bddfa 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/config/DefaultConfigurationStore.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/config/DefaultConfigurationStore.java @@ -28,16 +28,21 @@ public class DefaultConfigurationStore implements PolarisConfigurationStore { private final Map config; private final Map> realmConfig; + private final DynamicFeatureConfigResolver dynamicFeatureConfigResolver; public DefaultConfigurationStore(Map config) { this.config = config; this.realmConfig = Map.of(); + this.dynamicFeatureConfigResolver = new NoOpDynamicFeatureConfigResolver(); } public DefaultConfigurationStore( - Map config, Map> realmConfig) { + Map config, + Map> realmConfig, + DynamicFeatureConfigResolver dynamicFeatureConfigResolver) { this.config = config; this.realmConfig = realmConfig; + this.dynamicFeatureConfigResolver = dynamicFeatureConfigResolver; } @SuppressWarnings("unchecked") @@ -45,6 +50,11 @@ public DefaultConfigurationStore( public @Nullable T getConfiguration(@NotNull PolarisCallContext ctx, String configName) { String realm = CallContext.getCurrentContext().getRealmContext().getRealmIdentifier(); return (T) - realmConfig.getOrDefault(realm, Map.of()).getOrDefault(configName, config.get(configName)); + dynamicFeatureConfigResolver + .resolve(configName) + .orElse( + realmConfig + .getOrDefault(realm, Map.of()) + .getOrDefault(configName, config.get(configName))); } } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/config/DynamicFeatureConfigResolver.java b/polaris-service/src/main/java/org/apache/polaris/service/config/DynamicFeatureConfigResolver.java new file mode 100644 index 0000000000..25b3e4ea09 --- /dev/null +++ b/polaris-service/src/main/java/org/apache/polaris/service/config/DynamicFeatureConfigResolver.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.config; + +import io.dropwizard.jackson.Discoverable; +import java.util.Optional; + +public interface DynamicFeatureConfigResolver extends Discoverable { + Optional resolve(String key); +} diff --git a/polaris-service/src/main/java/org/apache/polaris/service/config/NoOpDynamicFeatureConfigResolver.java b/polaris-service/src/main/java/org/apache/polaris/service/config/NoOpDynamicFeatureConfigResolver.java new file mode 100644 index 0000000000..26e9cc50e9 --- /dev/null +++ b/polaris-service/src/main/java/org/apache/polaris/service/config/NoOpDynamicFeatureConfigResolver.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.config; + +import java.util.Optional; +import org.apache.polaris.core.PolarisConfiguration; + +public class NoOpDynamicFeatureConfigResolver implements DynamicFeatureConfigResolver { + @Override + public Optional resolve(PolarisConfiguration key) { + return Optional.empty(); + } +} diff --git a/polaris-service/src/main/java/org/apache/polaris/service/config/PolarisApplicationConfig.java b/polaris-service/src/main/java/org/apache/polaris/service/config/PolarisApplicationConfig.java index 7d5fd8fdf5..3a4b39e04b 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/config/PolarisApplicationConfig.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/config/PolarisApplicationConfig.java @@ -63,6 +63,7 @@ public class PolarisApplicationConfig extends Configuration { private String awsSecretKey; private FileIOFactory fileIOFactory; private RateLimiter rateLimiter; + private DynamicFeatureConfigResolver dynamicFeatureConfigResolver; private AccessToken gcpAccessToken; @@ -89,6 +90,17 @@ public FileIOFactory getFileIOFactory() { return fileIOFactory; } + @JsonProperty("dynamicFeatureConfigResolver") + public void setDynamicFeatureConfigResolver( + DynamicFeatureConfigResolver dynamicFeatureConfigResolver) { + this.dynamicFeatureConfigResolver = dynamicFeatureConfigResolver; + } + + @JsonProperty("dynamicFeatureConfigResolver") + public DynamicFeatureConfigResolver getDynamicFeatureConfigResolver() { + return dynamicFeatureConfigResolver; + } + @JsonProperty("authenticator") public void setPolarisAuthenticator( DiscoverableAuthenticator polarisAuthenticator) { @@ -194,7 +206,8 @@ public long getMaxRequestBodyBytes() { } public PolarisConfigurationStore getConfigurationStore() { - return new DefaultConfigurationStore(globalFeatureConfiguration, realmConfiguration); + return new DefaultConfigurationStore( + globalFeatureConfiguration, realmConfiguration, dynamicFeatureConfigResolver); } public List getDefaultRealms() { diff --git a/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.config.DynamicFeatureConfigResolver b/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.config.DynamicFeatureConfigResolver new file mode 100644 index 0000000000..e62bdda231 --- /dev/null +++ b/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.config.DynamicFeatureConfigResolver @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +org.apache.polaris.service.config.NoOpDynamicFeatureConfigResolver \ No newline at end of file diff --git a/polaris-service/src/test/java/org/apache/polaris/service/config/DefaultConfigurationStoreTest.java b/polaris-service/src/test/java/org/apache/polaris/service/config/DefaultConfigurationStoreTest.java index 7c981ab947..2a8bedbfdc 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/config/DefaultConfigurationStoreTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/config/DefaultConfigurationStoreTest.java @@ -65,7 +65,8 @@ public void testGetRealmConfiguration() { "realm1", Map.of("key1", realm1KeyOneValue), "realm2", - Map.of("key1", realm2KeyOneValue, "key2", realm2KeyTwoValue))); + Map.of("key1", realm2KeyOneValue, "key2", realm2KeyTwoValue)), + new NoOpDynamicFeatureConfigResolver()); InMemoryPolarisMetaStoreManagerFactory metastoreFactory = new InMemoryPolarisMetaStoreManagerFactory(); From 0fad69a88330573e0411a8628545682b2a4e0329 Mon Sep 17 00:00:00 2001 From: andrew4699 Date: Tue, 19 Nov 2024 15:52:32 -0800 Subject: [PATCH 2/6] Add javadoc --- .../service/config/DynamicFeatureConfigResolver.java | 11 +++++++++++ .../config/NoOpDynamicFeatureConfigResolver.java | 4 ++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/polaris-service/src/main/java/org/apache/polaris/service/config/DynamicFeatureConfigResolver.java b/polaris-service/src/main/java/org/apache/polaris/service/config/DynamicFeatureConfigResolver.java index 25b3e4ea09..7abb3c1c59 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/config/DynamicFeatureConfigResolver.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/config/DynamicFeatureConfigResolver.java @@ -21,6 +21,17 @@ import io.dropwizard.jackson.Discoverable; import java.util.Optional; +/** + * DynamicFeatureConfigResolvers dynamically resolve featureConfigurations. This is useful for + * integration with feature flag systems which are intended for fetching configs at runtime. + */ public interface DynamicFeatureConfigResolver extends Discoverable { + /** + * Resolves a dynamic config by its key name. + * + * @param key + * @return The config value or Optional.empty() if the config should not be dynamically resolved. + * If it's not dynamically resolved, it will be deferred to the application config. + */ Optional resolve(String key); } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/config/NoOpDynamicFeatureConfigResolver.java b/polaris-service/src/main/java/org/apache/polaris/service/config/NoOpDynamicFeatureConfigResolver.java index 26e9cc50e9..b575b70c9a 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/config/NoOpDynamicFeatureConfigResolver.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/config/NoOpDynamicFeatureConfigResolver.java @@ -19,11 +19,11 @@ package org.apache.polaris.service.config; import java.util.Optional; -import org.apache.polaris.core.PolarisConfiguration; +/** An empty dynamic config resolver. */ public class NoOpDynamicFeatureConfigResolver implements DynamicFeatureConfigResolver { @Override - public Optional resolve(PolarisConfiguration key) { + public Optional resolve(String key) { return Optional.empty(); } } From 43834f01a8618a784a00f181800f3f096f84bf4f Mon Sep 17 00:00:00 2001 From: andrew4699 Date: Tue, 19 Nov 2024 16:46:56 -0800 Subject: [PATCH 3/6] Test precedence --- .../config/DefaultConfigurationStoreTest.java | 79 ++++++++++++++++++- 1 file changed, 78 insertions(+), 1 deletion(-) diff --git a/polaris-service/src/test/java/org/apache/polaris/service/config/DefaultConfigurationStoreTest.java b/polaris-service/src/test/java/org/apache/polaris/service/config/DefaultConfigurationStoreTest.java index 2a8bedbfdc..721da64958 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/config/DefaultConfigurationStoreTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/config/DefaultConfigurationStoreTest.java @@ -20,14 +20,18 @@ import static org.assertj.core.api.Assertions.assertThat; +import java.util.HashMap; import java.util.Map; +import java.util.Optional; +import java.util.stream.Stream; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; public class DefaultConfigurationStoreTest { - @Test public void testGetConfiguration() { DefaultConfigurationStore defaultConfigurationStore = @@ -107,4 +111,77 @@ public void testGetRealmConfiguration() { String keyTwoRealm3 = defaultConfigurationStore.getConfiguration(realm3Ctx, "key2"); assertThat(keyTwoRealm3).isEqualTo(defaultKeyTwoValue); } + + @Test + public void testDynamicConfig() { + InMemoryPolarisMetaStoreManagerFactory metastoreFactory = + new InMemoryPolarisMetaStoreManagerFactory(); + PolarisCallContext polarisCtx = + new PolarisCallContext( + metastoreFactory.getOrCreateSessionSupplier(() -> "realm1").get(), + new PolarisDefaultDiagServiceImpl()); + + String key = "k1"; + Map staticConfig = Map.of(key, 10); + + assertThat( + new DefaultConfigurationStore(staticConfig, Map.of(), k -> Optional.empty()) + .getConfiguration(polarisCtx, key)) + .as("The DynamicFeatureConfigResolver always returns Optional.empty()") + .isEqualTo(10); + + assertThat( + new DefaultConfigurationStore(staticConfig, Map.of(), k -> Optional.of(5)) + .getConfiguration(polarisCtx, key)) + .as("The DynamicFeatureConfigResolver always returns 5") + .isEqualTo(5); + } + + @ParameterizedTest + @MethodSource("getTestConfigs") + public void testPrecedenceIsDynamicThenStaticPerRealmThenStaticGlobal(TestConfig testConfig) { + InMemoryPolarisMetaStoreManagerFactory metastoreFactory = + new InMemoryPolarisMetaStoreManagerFactory(); + + String realm = "realm1"; + PolarisCallContext polarisCtx = + new PolarisCallContext( + metastoreFactory.getOrCreateSessionSupplier(() -> realm).get(), + new PolarisDefaultDiagServiceImpl()); + + String key = "k1"; + + Map staticConfig = new HashMap<>(); + if (testConfig.staticConfig != null) { + staticConfig.put(key, testConfig.staticConfig); + } + + Map realmConfig = new HashMap<>(); + if (testConfig.realmConfig != null) { + realmConfig.put(key, testConfig.realmConfig); + } + + DefaultConfigurationStore configStore = + new DefaultConfigurationStore( + staticConfig, + Map.of(realm, realmConfig), + (k) -> Optional.ofNullable(testConfig.dynamicConfig)); + assertThat(configStore.getConfiguration(polarisCtx, key)) + .isEqualTo(testConfig.expectedValue); + } + + private static Stream getTestConfigs() { + return Stream.of( + new TestConfig(null, null, null, null), + new TestConfig(5, null, null, 5), + new TestConfig(5, 6, null, 6), + new TestConfig(5, 6, 7, 7), + new TestConfig(5, null, 7, 7), + new TestConfig(null, null, 7, 7), + new TestConfig(null, 6, 7, 7), + new TestConfig(null, 6, null, 6)); + } + + public record TestConfig( + Integer staticConfig, Integer realmConfig, Integer dynamicConfig, Integer expectedValue) {} } From 870afd2e0f8a59477c682ca560b0e12335a571bf Mon Sep 17 00:00:00 2001 From: andrew4699 Date: Tue, 19 Nov 2024 16:47:02 -0800 Subject: [PATCH 4/6] Add no-op resolver to config --- polaris-server.yml | 2 ++ .../service/config/NoOpDynamicFeatureConfigResolver.java | 2 ++ .../src/test/resources/polaris-server-integrationtest.yml | 2 ++ 3 files changed, 6 insertions(+) diff --git a/polaris-server.yml b/polaris-server.yml index e147d4d48e..5771b897fe 100644 --- a/polaris-server.yml +++ b/polaris-server.yml @@ -73,6 +73,8 @@ featureConfiguration: - AZURE - FILE +dynamicFeatureConfigResolver: no-op + callContextResolver: type: default diff --git a/polaris-service/src/main/java/org/apache/polaris/service/config/NoOpDynamicFeatureConfigResolver.java b/polaris-service/src/main/java/org/apache/polaris/service/config/NoOpDynamicFeatureConfigResolver.java index b575b70c9a..cbaa0ad9b6 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/config/NoOpDynamicFeatureConfigResolver.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/config/NoOpDynamicFeatureConfigResolver.java @@ -18,9 +18,11 @@ */ package org.apache.polaris.service.config; +import com.fasterxml.jackson.annotation.JsonTypeName; import java.util.Optional; /** An empty dynamic config resolver. */ +@JsonTypeName("no-op") public class NoOpDynamicFeatureConfigResolver implements DynamicFeatureConfigResolver { @Override public Optional resolve(String key) { diff --git a/polaris-service/src/test/resources/polaris-server-integrationtest.yml b/polaris-service/src/test/resources/polaris-server-integrationtest.yml index 10fd38d86b..55335827b1 100644 --- a/polaris-service/src/test/resources/polaris-server-integrationtest.yml +++ b/polaris-service/src/test/resources/polaris-server-integrationtest.yml @@ -79,6 +79,8 @@ featureConfiguration: - GCS - AZURE +dynamicFeatureConfigResolver: no-op + metaStoreManager: type: in-memory From 3fca70a52933dab5f130e5d0cc77097e0f374d9d Mon Sep 17 00:00:00 2001 From: andrew4699 Date: Wed, 20 Nov 2024 12:03:23 -0800 Subject: [PATCH 5/6] Try fixing Jackson type resolution --- polaris-server.yml | 3 ++- .../polaris/service/config/DynamicFeatureConfigResolver.java | 2 ++ .../src/test/resources/polaris-server-integrationtest.yml | 3 ++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/polaris-server.yml b/polaris-server.yml index 5771b897fe..74b4d5fd61 100644 --- a/polaris-server.yml +++ b/polaris-server.yml @@ -73,7 +73,8 @@ featureConfiguration: - AZURE - FILE -dynamicFeatureConfigResolver: no-op +dynamicFeatureConfigResolver: + type: no-op callContextResolver: type: default diff --git a/polaris-service/src/main/java/org/apache/polaris/service/config/DynamicFeatureConfigResolver.java b/polaris-service/src/main/java/org/apache/polaris/service/config/DynamicFeatureConfigResolver.java index 7abb3c1c59..ae0c692754 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/config/DynamicFeatureConfigResolver.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/config/DynamicFeatureConfigResolver.java @@ -18,6 +18,7 @@ */ package org.apache.polaris.service.config; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import io.dropwizard.jackson.Discoverable; import java.util.Optional; @@ -25,6 +26,7 @@ * DynamicFeatureConfigResolvers dynamically resolve featureConfigurations. This is useful for * integration with feature flag systems which are intended for fetching configs at runtime. */ +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") public interface DynamicFeatureConfigResolver extends Discoverable { /** * Resolves a dynamic config by its key name. diff --git a/polaris-service/src/test/resources/polaris-server-integrationtest.yml b/polaris-service/src/test/resources/polaris-server-integrationtest.yml index 55335827b1..207ca6784a 100644 --- a/polaris-service/src/test/resources/polaris-server-integrationtest.yml +++ b/polaris-service/src/test/resources/polaris-server-integrationtest.yml @@ -79,7 +79,8 @@ featureConfiguration: - GCS - AZURE -dynamicFeatureConfigResolver: no-op +dynamicFeatureConfigResolver: + type: no-op metaStoreManager: type: in-memory From 16f56f7fa10b10d23125e1ae846f29d47a759d27 Mon Sep 17 00:00:00 2001 From: andrew4699 Date: Wed, 20 Nov 2024 12:12:16 -0800 Subject: [PATCH 6/6] Add to Discoverable META-INF --- .../META-INF/services/io.dropwizard.jackson.Discoverable | 1 + 1 file changed, 1 insertion(+) diff --git a/polaris-service/src/main/resources/META-INF/services/io.dropwizard.jackson.Discoverable b/polaris-service/src/main/resources/META-INF/services/io.dropwizard.jackson.Discoverable index 95d1f8ec7a..b26bf19c23 100644 --- a/polaris-service/src/main/resources/META-INF/services/io.dropwizard.jackson.Discoverable +++ b/polaris-service/src/main/resources/META-INF/services/io.dropwizard.jackson.Discoverable @@ -19,6 +19,7 @@ org.apache.polaris.service.auth.DiscoverableAuthenticator org.apache.polaris.core.persistence.MetaStoreManagerFactory +org.apache.polaris.service.config.DynamicFeatureConfigResolver org.apache.polaris.service.config.OAuth2ApiService org.apache.polaris.service.context.RealmContextResolver org.apache.polaris.service.context.CallContextResolver