From 800b946dc2669d8331cb7c25883bafbdd4b0a18e Mon Sep 17 00:00:00 2001 From: Prashant Date: Wed, 23 Apr 2025 10:45:28 -0700 Subject: [PATCH 01/11] Plumb admin pending Service E2E --- .../relational-jdbc/build.gradle.kts | 5 +- .../jdbc/JdbcMetaStoreManagerFactory.java | 14 ++- .../relational/jdbc/models/ModelEntity.java | 4 +- .../postgres/schema-v1-postgresql.sql | 15 ++-- quarkus/admin/build.gradle.kts | 2 + .../RelationalJdbcBootstrapCommandTest.java | 26 ++++++ .../jdbc/RelationalJdbcProfile.java | 41 +++++++++ .../jdbc/RelationalJdbcPurgeCommandTest.java | 38 ++++++++ ...gresRelationalJdbcLifeCycleManagement.java | 86 +++++++++++++++++++ quarkus/server/build.gradle.kts | 2 + quarkus/service/build.gradle.kts | 16 +++- .../jdbc/JdbcQuarkusApplicationIT.java | 29 +++++++ .../jdbc/RelationalJdbcProfile.java | 40 +++++++++ ...gresRelationalJdbcLifeCycleManagement.java | 86 +++++++++++++++++++ 14 files changed, 380 insertions(+), 24 deletions(-) rename {scripts => extension/persistence/relational-jdbc/src/main/resources}/postgres/schema-v1-postgresql.sql (92%) create mode 100644 quarkus/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcBootstrapCommandTest.java create mode 100644 quarkus/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcProfile.java create mode 100644 quarkus/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcPurgeCommandTest.java create mode 100644 quarkus/admin/src/testFixtures/java/org/apache/polaris/admintool/PostgresRelationalJdbcLifeCycleManagement.java create mode 100644 quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusApplicationIT.java create mode 100644 quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/RelationalJdbcProfile.java create mode 100644 quarkus/service/src/testFixtures/java/org/apache/polaris/service/quarkus/it/PostgresRelationalJdbcLifeCycleManagement.java diff --git a/extension/persistence/relational-jdbc/build.gradle.kts b/extension/persistence/relational-jdbc/build.gradle.kts index 0895d21ec0..82f67c8a5a 100644 --- a/extension/persistence/relational-jdbc/build.gradle.kts +++ b/extension/persistence/relational-jdbc/build.gradle.kts @@ -17,7 +17,10 @@ * under the License. */ -plugins { id("polaris-server") } +plugins { + id("polaris-server") + alias(libs.plugins.jandex) +} dependencies { implementation(project(":polaris-core")) diff --git a/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java b/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java index 07ac7f9725..052af57693 100644 --- a/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java +++ b/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java @@ -21,6 +21,7 @@ import io.smallrye.common.annotation.Identifier; import jakarta.annotation.Nullable; import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.inject.Instance; import jakarta.inject.Inject; import java.sql.SQLException; import java.util.HashMap; @@ -37,10 +38,7 @@ import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PolarisPrincipalSecrets; -import org.apache.polaris.core.persistence.BasePersistence; -import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.core.persistence.PrincipalSecretsGenerator; +import org.apache.polaris.core.persistence.*; import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; import org.apache.polaris.core.persistence.cache.EntityCache; import org.apache.polaris.core.persistence.dao.entity.BaseResult; @@ -70,7 +68,7 @@ public class JdbcMetaStoreManagerFactory implements MetaStoreManagerFactory { protected final PolarisDiagnostics diagServices = new PolarisDefaultDiagServiceImpl(); // TODO: Pending discussion of if we should have one Database per realm or 1 schema per realm // or realm should be a primary key on all the tables. - @Inject DataSource dataSource; + @Inject Instance dataSource; @Inject PolarisStorageIntegrationProvider storageIntegrationProvider; protected JdbcMetaStoreManagerFactory() {} @@ -86,7 +84,7 @@ protected PrincipalSecretsGenerator secretsGenerator( } protected PolarisMetaStoreManager createNewMetaStoreManager() { - return new TransactionalMetaStoreManagerImpl(); + return new AtomicOperationMetaStoreManager(); } private void initializeForRealm( @@ -106,12 +104,12 @@ private void initializeForRealm( } private DatasourceOperations getDatasourceOperations(boolean isBootstrap) { - DatasourceOperations databaseOperations = new DatasourceOperations(dataSource); + DatasourceOperations databaseOperations = new DatasourceOperations(dataSource.get()); if (isBootstrap) { // TODO: see if we need to take script from Quarkus or can we just // use the script committed in the repo. try { - databaseOperations.executeScript("scripts/postgres/schema-v1-postgres.sql"); + databaseOperations.executeScript("postgres/schema-v1-postgresql.sql"); } catch (SQLException e) { throw new RuntimeException( String.format("Error executing sql script: %s", e.getMessage()), e); diff --git a/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/models/ModelEntity.java b/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/models/ModelEntity.java index 5ec7bbf95a..92f6e35c77 100644 --- a/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/models/ModelEntity.java +++ b/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/models/ModelEntity.java @@ -152,8 +152,8 @@ public ModelEntity fromResultSet(ResultSet r) throws SQLException { .purgeTimestamp(r.getObject("purge_timestamp", Long.class)) .toPurgeTimestamp(r.getObject("to_purge_timestamp", Long.class)) .lastUpdateTimestamp(r.getObject("last_update_timestamp", Long.class)) - .properties(r.getObject("properties", String.class)) - .internalProperties(r.getObject("internal_properties", String.class)) + .properties(r.getString("properties")) + .internalProperties(r.getString("internal_properties")) .grantRecordsVersion(r.getObject("grant_records_version", Integer.class)) .build(); } diff --git a/scripts/postgres/schema-v1-postgresql.sql b/extension/persistence/relational-jdbc/src/main/resources/postgres/schema-v1-postgresql.sql similarity index 92% rename from scripts/postgres/schema-v1-postgresql.sql rename to extension/persistence/relational-jdbc/src/main/resources/postgres/schema-v1-postgresql.sql index 3e18d9e30a..12da8f7a84 100644 --- a/scripts/postgres/schema-v1-postgresql.sql +++ b/extension/persistence/relational-jdbc/src/main/resources/postgres/schema-v1-postgresql.sql @@ -15,16 +15,12 @@ -- KIND, either express or implied. See the License for the -- specific language governing permissions and limitations -- under the License. --- --- Note: Database and schema creation is not included in this script. Please create the database and --- schema before running this script. for example in psql: --- CREATE DATABASE polaris_db; --- \c polaris_db --- CREATE SCHEMA polaris_schema; --- set search_path to polaris_schema; +CREATE SCHEMA IF NOT EXISTS POLARIS_SCHEMA; +SET search_path TO POLARIS_SCHEMA; CREATE TABLE IF NOT EXISTS entities ( + realm_id TEXT NOT NULL, catalog_id BIGINT NOT NULL, id BIGINT NOT NULL, parent_id BIGINT NOT NULL, @@ -49,6 +45,7 @@ CREATE INDEX IF NOT EXISTS idx_entities ON entities (realm_id, catalog_id, id); COMMENT ON TABLE entities IS 'all the entities'; +COMMENT ON COLUMN entities.realm_id IS 'realm_id used for multi-tenancy'; COMMENT ON COLUMN entities.catalog_id IS 'catalog id'; COMMENT ON COLUMN entities.id IS 'entity id'; COMMENT ON COLUMN entities.parent_id IS 'entity id of parent'; @@ -65,7 +62,7 @@ COMMENT ON COLUMN entities.internal_properties IS 'entities internal properties COMMENT ON COLUMN entities.grant_records_version IS 'the version of grant records change on the entity'; CREATE TABLE IF NOT EXISTS grant_records ( - realm_id INT NOT NULL, + realm_id TEXT NOT NULL, securable_catalog_id BIGINT NOT NULL, securable_id BIGINT NOT NULL, grantee_catalog_id BIGINT NOT NULL, @@ -84,7 +81,7 @@ COMMENT ON COLUMN grant_records.privilege_code IS 'privilege code'; CREATE TABLE IF NOT EXISTS principal_authentication_data ( - realm_id INT NOT NULL, + realm_id TEXT NOT NULL, principal_id BIGINT NOT NULL, principal_client_id VARCHAR(255) NOT NULL, main_secret_hash VARCHAR(255) NOT NULL, diff --git a/quarkus/admin/build.gradle.kts b/quarkus/admin/build.gradle.kts index 357acfd945..c7f281bc14 100644 --- a/quarkus/admin/build.gradle.kts +++ b/quarkus/admin/build.gradle.kts @@ -44,8 +44,10 @@ dependencies { implementation(project(":polaris-api-iceberg-service")) runtimeOnly(project(":polaris-eclipselink")) + runtimeOnly(project(":polaris-relational-jdbc")) runtimeOnly("org.postgresql:postgresql") + implementation("io.quarkus:quarkus-jdbc-postgresql") implementation(enforcedPlatform(libs.quarkus.bom)) implementation("io.quarkus:quarkus-picocli") implementation("io.quarkus:quarkus-container-image-docker") diff --git a/quarkus/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcBootstrapCommandTest.java b/quarkus/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcBootstrapCommandTest.java new file mode 100644 index 0000000000..9cf0ed3d92 --- /dev/null +++ b/quarkus/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcBootstrapCommandTest.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.admintool.relational.jdbc; + +import io.quarkus.test.junit.TestProfile; +import org.apache.polaris.admintool.BootstrapCommandTestBase; + +@TestProfile(RelationalJdbcProfile.class) +public class RelationalJdbcBootstrapCommandTest extends BootstrapCommandTestBase {} diff --git a/quarkus/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcProfile.java b/quarkus/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcProfile.java new file mode 100644 index 0000000000..652e4c4d2f --- /dev/null +++ b/quarkus/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcProfile.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.admintool.relational.jdbc; + +import static org.apache.polaris.admintool.PostgresTestResourceLifecycleManager.INIT_SCRIPT; + +import io.quarkus.test.junit.QuarkusTestProfile; +import java.util.List; +import java.util.Map; +import org.apache.polaris.admintool.PostgresRelationalJdbcLifeCycleManagement; + +public class RelationalJdbcProfile implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + return Map.of(); + } + + @Override + public List testResources() { + return List.of( + new TestResourceEntry( + PostgresRelationalJdbcLifeCycleManagement.class, + Map.of(INIT_SCRIPT, "org/apache/polaris/admintool/init.sql"))); + } +} diff --git a/quarkus/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcPurgeCommandTest.java b/quarkus/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcPurgeCommandTest.java new file mode 100644 index 0000000000..a8c123885f --- /dev/null +++ b/quarkus/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcPurgeCommandTest.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.admintool.relational.jdbc; + +import io.quarkus.test.junit.TestProfile; +import java.util.Map; +import org.apache.polaris.admintool.PurgeCommandTestBase; +import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; + +@TestProfile(RelationalJdbcPurgeCommandTest.Profile.class) +public class RelationalJdbcPurgeCommandTest extends PurgeCommandTestBase { + public static class Profile extends RelationalJdbcProfile { + @Override + public Map getConfigOverrides() { + return ImmutableMap.builder() + .putAll(super.getConfigOverrides()) + .put("pre-bootstrap", "true") + .build(); + } + } +} \ No newline at end of file diff --git a/quarkus/admin/src/testFixtures/java/org/apache/polaris/admintool/PostgresRelationalJdbcLifeCycleManagement.java b/quarkus/admin/src/testFixtures/java/org/apache/polaris/admintool/PostgresRelationalJdbcLifeCycleManagement.java new file mode 100644 index 0000000000..6a983f3781 --- /dev/null +++ b/quarkus/admin/src/testFixtures/java/org/apache/polaris/admintool/PostgresRelationalJdbcLifeCycleManagement.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.admintool; + +import io.quarkus.test.common.DevServicesContext; +import io.quarkus.test.common.QuarkusTestResourceLifecycleManager; +import java.util.Map; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.utility.DockerImageName; + +public class PostgresRelationalJdbcLifeCycleManagement + implements QuarkusTestResourceLifecycleManager, DevServicesContext.ContextAware { + private static final String INIT_SCRIPT = "init-script"; + + private PostgreSQLContainer postgres; + private String initScript; + private DevServicesContext context; + + @Override + public void init(Map initArgs) { + initScript = initArgs.get(INIT_SCRIPT); + } + + @Override + @SuppressWarnings("resource") + public Map start() { + postgres = + new PostgreSQLContainer<>(DockerImageName.parse("postgres:17")) + .withDatabaseName("polaris_db") + .withUsername("polaris") + .withPassword("polaris"); + + if (initScript != null) { + postgres.withInitScript(initScript); + } + + context.containerNetworkId().ifPresent(postgres::withNetworkMode); + postgres.start(); + return Map.of( + "polaris.persistence.type", + "relational-jdbc", + "quarkus.datasource.db-kind", + "pgsql", + "quarkus.datasource.jdbc.url", + postgres.getJdbcUrl(), + "quarkus.datasource.username", + postgres.getUsername(), + "quarkus.datasource.password", + postgres.getPassword(), + "quarkus.datasource.jdbc.initial-size", + "10"); + } + + @Override + public void stop() { + if (postgres != null) { + try { + postgres.stop(); + } finally { + postgres = null; + } + } + } + + @Override + public void setIntegrationTestContext(DevServicesContext context) { + this.context = context; + } +} \ No newline at end of file diff --git a/quarkus/server/build.gradle.kts b/quarkus/server/build.gradle.kts index 61c1b46660..9e93b4cdb0 100644 --- a/quarkus/server/build.gradle.kts +++ b/quarkus/server/build.gradle.kts @@ -50,6 +50,8 @@ dependencies { runtimeOnly(project(":polaris-eclipselink")) runtimeOnly("org.postgresql:postgresql") + runtimeOnly(project(":polaris-relational-jdbc")) + implementation("io.quarkus:quarkus-jdbc-postgresql") // enforce the Quarkus _platform_ here, to get a consistent and validated set of dependencies implementation(enforcedPlatform(libs.quarkus.bom)) diff --git a/quarkus/service/build.gradle.kts b/quarkus/service/build.gradle.kts index eb729b0204..1ed04a5556 100644 --- a/quarkus/service/build.gradle.kts +++ b/quarkus/service/build.gradle.kts @@ -23,10 +23,10 @@ plugins { id("polaris-quarkus") } -configurations.all { - // exclude junit4 dependency for this module - exclude(group = "junit", module = "junit") -} +//configurations.all { +// // exclude junit4 dependency for this module +// exclude(group = "junit", module = "junit") +//} dependencies { implementation(project(":polaris-core")) @@ -113,6 +113,7 @@ dependencies { testImplementation("software.amazon.awssdk:kms") testImplementation("software.amazon.awssdk:dynamodb") + runtimeOnly(project(":polaris-relational-jdbc")) testImplementation(platform(libs.quarkus.bom)) testImplementation("io.quarkus:quarkus-junit5") testImplementation("io.quarkus:quarkus-junit5-mockito") @@ -122,6 +123,13 @@ dependencies { testImplementation(libs.threeten.extra) testImplementation(libs.hawkular.agent.prometheus.scraper) + + testFixturesApi(enforcedPlatform(libs.quarkus.bom)) + testFixturesApi("io.quarkus:quarkus-junit5") + testFixturesApi(platform(libs.testcontainers.bom)) + testFixturesApi("org.testcontainers:testcontainers") + testFixturesApi("org.testcontainers:postgresql") + testFixturesApi("org.postgresql:postgresql") } tasks.withType(Test::class.java).configureEach { diff --git a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusApplicationIT.java b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusApplicationIT.java new file mode 100644 index 0000000000..0829146b37 --- /dev/null +++ b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusApplicationIT.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.quarkus.it.relational.jdbc; + +import io.quarkus.test.junit.QuarkusIntegrationTest; +import io.quarkus.test.junit.TestProfile; +import org.apache.polaris.service.it.test.PolarisApplicationIntegrationTest; + +@TestProfile(RelationalJdbcProfile.class) +@QuarkusIntegrationTest +public class JdbcQuarkusApplicationIT extends PolarisApplicationIntegrationTest { +} diff --git a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/RelationalJdbcProfile.java b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/RelationalJdbcProfile.java new file mode 100644 index 0000000000..ac52da2d51 --- /dev/null +++ b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/RelationalJdbcProfile.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.quarkus.it.relational.jdbc; + +import io.quarkus.test.junit.QuarkusTestProfile; +import org.apache.polaris.service.quarkus.it.PostgresRelationalJdbcLifeCycleManagement; + +import java.util.List; +import java.util.Map; + +public class RelationalJdbcProfile implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + return Map.of(); + } + + @Override + public List testResources() { + return List.of( + new TestResourceEntry( + PostgresRelationalJdbcLifeCycleManagement.class, Map.of())); + } +} diff --git a/quarkus/service/src/testFixtures/java/org/apache/polaris/service/quarkus/it/PostgresRelationalJdbcLifeCycleManagement.java b/quarkus/service/src/testFixtures/java/org/apache/polaris/service/quarkus/it/PostgresRelationalJdbcLifeCycleManagement.java new file mode 100644 index 0000000000..1c3835e011 --- /dev/null +++ b/quarkus/service/src/testFixtures/java/org/apache/polaris/service/quarkus/it/PostgresRelationalJdbcLifeCycleManagement.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.quarkus.it; + +import io.quarkus.test.common.DevServicesContext; +import io.quarkus.test.common.QuarkusTestResourceLifecycleManager; +import java.util.Map; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.utility.DockerImageName; + +public class PostgresRelationalJdbcLifeCycleManagement + implements QuarkusTestResourceLifecycleManager, DevServicesContext.ContextAware { + private static final String INIT_SCRIPT = "init-script"; + + private PostgreSQLContainer postgres; + private String initScript; + private DevServicesContext context; + + @Override + public void init(Map initArgs) { + initScript = initArgs.get(INIT_SCRIPT); + } + + @Override + @SuppressWarnings("resource") + public Map start() { + postgres = + new PostgreSQLContainer<>(DockerImageName.parse("postgres:17")) + .withDatabaseName("polaris_db") + .withUsername("polaris") + .withPassword("polaris"); + + if (initScript != null) { + postgres.withInitScript(initScript); + } + + context.containerNetworkId().ifPresent(postgres::withNetworkMode); + postgres.start(); + return Map.of( + "polaris.persistence.type", + "relational-jdbc", + "quarkus.datasource.db-kind", + "pgsql", + "quarkus.datasource.jdbc.url", + postgres.getJdbcUrl(), + "quarkus.datasource.username", + postgres.getUsername(), + "quarkus.datasource.password", + postgres.getPassword(), + "quarkus.datasource.jdbc.initial-size", + "10"); + } + + @Override + public void stop() { + if (postgres != null) { + try { + postgres.stop(); + } finally { + postgres = null; + } + } + } + + @Override + public void setIntegrationTestContext(DevServicesContext context) { + this.context = context; + } +} From 530c9d61cb7121872aaf443c6938db8ef5e01b09 Mon Sep 17 00:00:00 2001 From: Prashant Date: Wed, 23 Apr 2025 13:21:17 -0700 Subject: [PATCH 02/11] Add E2E test for service --- .../jdbc/JdbcMetaStoreManagerFactory.java | 7 +++-- quarkus/service/build.gradle.kts | 1 + .../jdbc/JdbcQuarkusApplicationIT.java | 3 +- .../jdbc/JdbcQuarkusManagementServiceIT.java | 29 ++++++++++++++++++ .../jdbc/JdbcQuarkusRestCatalogViewAwsIT.java | 29 ++++++++++++++++++ .../JdbcQuarkusRestCatalogViewAzureIT.java | 29 ++++++++++++++++++ .../jdbc/JdbcQuarkusRestCatalogViewGcpIT.java | 30 +++++++++++++++++++ .../jdbc/JdbcQuarkusViewFileIT.java | 29 ++++++++++++++++++ .../it/relational/jdbc/JdbcRestCatalogIT.java | 29 ++++++++++++++++++ .../jdbc/RelationalJdbcProfile.java | 4 ++- .../quarkus/config/QuarkusProducers.java | 4 +-- 11 files changed, 188 insertions(+), 6 deletions(-) create mode 100644 quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusManagementServiceIT.java create mode 100644 quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusRestCatalogViewAwsIT.java create mode 100644 quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusRestCatalogViewAzureIT.java create mode 100644 quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusRestCatalogViewGcpIT.java create mode 100644 quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusViewFileIT.java create mode 100644 quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcRestCatalogIT.java diff --git a/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java b/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java index 052af57693..06c506543a 100644 --- a/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java +++ b/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java @@ -38,13 +38,16 @@ import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PolarisPrincipalSecrets; -import org.apache.polaris.core.persistence.*; +import org.apache.polaris.core.persistence.BasePersistence; +import org.apache.polaris.core.persistence.MetaStoreManagerFactory; +import org.apache.polaris.core.persistence.PolarisMetaStoreManager; +import org.apache.polaris.core.persistence.PrincipalSecretsGenerator; import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; import org.apache.polaris.core.persistence.cache.EntityCache; import org.apache.polaris.core.persistence.dao.entity.BaseResult; import org.apache.polaris.core.persistence.dao.entity.EntityResult; import org.apache.polaris.core.persistence.dao.entity.PrincipalSecretsResult; -import org.apache.polaris.core.persistence.transactional.TransactionalMetaStoreManagerImpl; +import org.apache.polaris.core.persistence.AtomicOperationMetaStoreManager; import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; import org.apache.polaris.core.storage.cache.StorageCredentialCache; import org.slf4j.Logger; diff --git a/quarkus/service/build.gradle.kts b/quarkus/service/build.gradle.kts index 1ed04a5556..f5fb7c5aed 100644 --- a/quarkus/service/build.gradle.kts +++ b/quarkus/service/build.gradle.kts @@ -114,6 +114,7 @@ dependencies { testImplementation("software.amazon.awssdk:dynamodb") runtimeOnly(project(":polaris-relational-jdbc")) + implementation("io.quarkus:quarkus-jdbc-postgresql") testImplementation(platform(libs.quarkus.bom)) testImplementation("io.quarkus:quarkus-junit5") testImplementation("io.quarkus:quarkus-junit5-mockito") diff --git a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusApplicationIT.java b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusApplicationIT.java index 0829146b37..0bcf15b183 100644 --- a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusApplicationIT.java +++ b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusApplicationIT.java @@ -20,10 +20,11 @@ package org.apache.polaris.service.quarkus.it.relational.jdbc; import io.quarkus.test.junit.QuarkusIntegrationTest; +import io.quarkus.test.junit.QuarkusTest; import io.quarkus.test.junit.TestProfile; import org.apache.polaris.service.it.test.PolarisApplicationIntegrationTest; @TestProfile(RelationalJdbcProfile.class) -@QuarkusIntegrationTest +@QuarkusTest public class JdbcQuarkusApplicationIT extends PolarisApplicationIntegrationTest { } diff --git a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusManagementServiceIT.java b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusManagementServiceIT.java new file mode 100644 index 0000000000..6e78f5d5fb --- /dev/null +++ b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusManagementServiceIT.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.quarkus.it.relational.jdbc; + +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.TestProfile; +import org.apache.polaris.service.it.test.PolarisManagementServiceIntegrationTest; + +@TestProfile(RelationalJdbcProfile.class) +@QuarkusTest +public class JdbcQuarkusManagementServiceIT extends PolarisManagementServiceIntegrationTest { +} diff --git a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusRestCatalogViewAwsIT.java b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusRestCatalogViewAwsIT.java new file mode 100644 index 0000000000..4fb79fad8f --- /dev/null +++ b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusRestCatalogViewAwsIT.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.quarkus.it.relational.jdbc; + +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.TestProfile; +import org.apache.polaris.service.it.test.PolarisRestCatalogViewAwsIntegrationTest; + +@TestProfile(RelationalJdbcProfile.class) +@QuarkusTest +public class JdbcQuarkusRestCatalogViewAwsIT extends PolarisRestCatalogViewAwsIntegrationTest { +} diff --git a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusRestCatalogViewAzureIT.java b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusRestCatalogViewAzureIT.java new file mode 100644 index 0000000000..a291e42097 --- /dev/null +++ b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusRestCatalogViewAzureIT.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.quarkus.it.relational.jdbc; + +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.TestProfile; +import org.apache.polaris.service.it.test.PolarisRestCatalogViewAzureIntegrationTest; + +@TestProfile(RelationalJdbcProfile.class) +@QuarkusTest +public class JdbcQuarkusRestCatalogViewAzureIT extends PolarisRestCatalogViewAzureIntegrationTest { +} diff --git a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusRestCatalogViewGcpIT.java b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusRestCatalogViewGcpIT.java new file mode 100644 index 0000000000..0bd60eee9e --- /dev/null +++ b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusRestCatalogViewGcpIT.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.quarkus.it.relational.jdbc; + +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.TestProfile; +import org.apache.polaris.service.it.test.PolarisRestCatalogViewGcpIntegrationTest; +import org.apache.polaris.service.quarkus.it.QuarkusRestCatalogViewGcpIT; + +@TestProfile(RelationalJdbcProfile.class) +@QuarkusTest +public class JdbcQuarkusRestCatalogViewGcpIT extends PolarisRestCatalogViewGcpIntegrationTest { +} diff --git a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusViewFileIT.java b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusViewFileIT.java new file mode 100644 index 0000000000..2cf23a0a43 --- /dev/null +++ b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusViewFileIT.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.quarkus.it.relational.jdbc; + +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.TestProfile; +import org.apache.polaris.service.it.test.PolarisRestCatalogViewFileIntegrationTest; + +@TestProfile(RelationalJdbcProfile.class) +@QuarkusTest +public class JdbcQuarkusViewFileIT extends PolarisRestCatalogViewFileIntegrationTest { +} diff --git a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcRestCatalogIT.java b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcRestCatalogIT.java new file mode 100644 index 0000000000..9ebf0728eb --- /dev/null +++ b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcRestCatalogIT.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.quarkus.it.relational.jdbc; + +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.TestProfile; +import org.apache.polaris.service.it.test.PolarisRestCatalogIntegrationTest; + +@TestProfile(RelationalJdbcProfile.class) +@QuarkusTest +public class JdbcRestCatalogIT extends PolarisRestCatalogIntegrationTest { +} diff --git a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/RelationalJdbcProfile.java b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/RelationalJdbcProfile.java index ac52da2d51..f54f4659ad 100644 --- a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/RelationalJdbcProfile.java +++ b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/RelationalJdbcProfile.java @@ -28,7 +28,9 @@ public class RelationalJdbcProfile implements QuarkusTestProfile { @Override public Map getConfigOverrides() { - return Map.of(); + return Map.of( + "polaris.persistence.auto-bootstrap-types", "relational-jdbc" + ); } @Override diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java index 625e57e45d..d7bfa74020 100644 --- a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java @@ -32,6 +32,7 @@ import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.core.Context; import java.time.Clock; + import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.PolarisDiagnostics; @@ -73,7 +74,6 @@ import org.eclipse.microprofile.context.ThreadContext; public class QuarkusProducers { - @Produces @ApplicationScoped // cannot be singleton because it is mocked in tests public Clock clock() { @@ -256,7 +256,7 @@ public ActiveRolesProvider activeRolesProvider( @ConfigProperty(name = "polaris.active-roles-provider.type") String persistenceType, @Any Instance activeRolesProviders) { return activeRolesProviders.select(Identifier.Literal.of(persistenceType)).get(); - } + } public void closeTaskExecutor(@Disposes @Identifier("task-executor") ManagedExecutor executor) { executor.close(); From eece61a21d78bd1f2b22ba65ab2635bbe79fa2d3 Mon Sep 17 00:00:00 2001 From: Prashant Date: Wed, 23 Apr 2025 13:38:27 -0700 Subject: [PATCH 03/11] fix bug --- .../jdbc/JdbcBasePersistenceImpl.java | 33 ++++-- .../jdbc/JdbcMetaStoreManagerFactory.java | 25 +++-- .../h2/{schema-v1-h2.sql => schema-v1.sql} | 0 ...schema-v1-postgresql.sql => schema-v1.sql} | 0 ...anagerWithJdbcBasePersistenceImplTest.java | 2 +- .../PolarisRestCatalogIntegrationTest.java | 6 ++ ...PolarisRestCatalogViewIntegrationBase.java | 8 +- .../admintool/config/QuarkusProducers.java | 27 +++++ .../RelationalJdbcBootstrapCommandTest.java | 1 - .../jdbc/RelationalJdbcProfile.java | 22 ++-- .../jdbc/RelationalJdbcPurgeCommandTest.java | 19 ++-- ...gresRelationalJdbcLifeCycleManagement.java | 101 +++++++++--------- quarkus/server/build.gradle.kts | 6 +- quarkus/service/build.gradle.kts | 17 +-- .../jdbc/JdbcQuarkusApplicationIT.java | 7 +- .../jdbc/JdbcQuarkusManagementServiceIT.java | 8 +- .../jdbc/JdbcQuarkusRestCatalogViewAwsIT.java | 8 +- .../JdbcQuarkusRestCatalogViewAzureIT.java | 8 +- .../jdbc/JdbcQuarkusRestCatalogViewGcpIT.java | 9 +- .../jdbc/JdbcQuarkusViewFileIT.java | 8 +- .../it/relational/jdbc/JdbcRestCatalogIT.java | 8 +- .../jdbc/RelationalJdbcProfile.java | 25 ++--- .../quarkus/config/QuarkusProducers.java | 31 +++++- .../quarkus/catalog/IcebergCatalogTest.java | 5 + .../catalog/IcebergCatalogViewTest.java | 5 + ...gresRelationalJdbcLifeCycleManagement.java | 99 +++++++++-------- 26 files changed, 286 insertions(+), 202 deletions(-) rename extension/persistence/relational-jdbc/src/main/resources/h2/{schema-v1-h2.sql => schema-v1.sql} (100%) rename extension/persistence/relational-jdbc/src/main/resources/postgres/{schema-v1-postgresql.sql => schema-v1.sql} (100%) diff --git a/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcBasePersistenceImpl.java b/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcBasePersistenceImpl.java index 24bb927027..dcbbaff8bb 100644 --- a/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcBasePersistenceImpl.java +++ b/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcBasePersistenceImpl.java @@ -93,8 +93,16 @@ public void writeEntity( query = generateInsertQuery(modelEntity, realmId); datasourceOperations.executeUpdate(query); } catch (SQLException e) { - if ((datasourceOperations.isConstraintViolation(e) - || datasourceOperations.isAlreadyExistsException(e))) { + if (datasourceOperations.isConstraintViolation(e)) { + PolarisBaseEntity existingEntity = + lookupEntityByName( + callCtx, + entity.getCatalogId(), + entity.getParentId(), + entity.getTypeCode(), + entity.getName()); + throw new EntityAlreadyExistsException(existingEntity, e); + } else if (datasourceOperations.isAlreadyExistsException(e)) { throw new EntityAlreadyExistsException(entity, e); } else { throw new RuntimeException( @@ -138,13 +146,14 @@ public void writeEntities( for (int i = 0; i < entities.size(); i++) { PolarisBaseEntity entity = entities.get(i); ModelEntity modelEntity = ModelEntity.fromEntity(entity); + boolean isUpdate = originalEntities != null && originalEntities.get(i) != null; // first, check if the entity has already been created, in which case we will simply // return it. PolarisBaseEntity entityFound = lookupEntity( callCtx, entity.getCatalogId(), entity.getId(), entity.getTypeCode()); - if (entityFound != null) { + if (entityFound != null && !isUpdate) { // probably the client retried, simply return it // TODO: Check correctness of returning entityFound vs entity here. It may have // already been updated after the creation. @@ -158,17 +167,25 @@ public void writeEntities( entity.getParentId(), entity.getTypeCode(), entity.getName()); - if (exists != null) { + if (exists != null && !isUpdate) { throw new EntityAlreadyExistsException(entity); } String query; - if (originalEntities == null || originalEntities.get(i) == null) { + if (!isUpdate) { try { query = generateInsertQuery(modelEntity, realmId); statement.executeUpdate(query); } catch (SQLException e) { - if ((datasourceOperations.isConstraintViolation(e) - || datasourceOperations.isAlreadyExistsException(e))) { + if (datasourceOperations.isConstraintViolation(e)) { + PolarisBaseEntity existingEntity = + lookupEntityByName( + callCtx, + entity.getCatalogId(), + entity.getParentId(), + entity.getTypeCode(), + entity.getName()); + throw new EntityAlreadyExistsException(existingEntity, e); + } else if (datasourceOperations.isAlreadyExistsException(e)) { throw new EntityAlreadyExistsException(entity, e); } else { throw new RuntimeException( @@ -492,6 +509,8 @@ public PolarisGrantRecord lookupGrantRecord( throw new IllegalStateException( String.format( "More than one grant record %s for a given Grant record", results.getFirst())); + } else if (results.isEmpty()) { + return null; } return results.getFirst(); } catch (SQLException e) { diff --git a/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java b/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java index 06c506543a..04175fb2b3 100644 --- a/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java +++ b/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java @@ -21,7 +21,6 @@ import io.smallrye.common.annotation.Identifier; import jakarta.annotation.Nullable; import jakarta.enterprise.context.ApplicationScoped; -import jakarta.enterprise.inject.Instance; import jakarta.inject.Inject; import java.sql.SQLException; import java.util.HashMap; @@ -38,6 +37,7 @@ import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PolarisPrincipalSecrets; +import org.apache.polaris.core.persistence.AtomicOperationMetaStoreManager; import org.apache.polaris.core.persistence.BasePersistence; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; @@ -47,7 +47,6 @@ import org.apache.polaris.core.persistence.dao.entity.BaseResult; import org.apache.polaris.core.persistence.dao.entity.EntityResult; import org.apache.polaris.core.persistence.dao.entity.PrincipalSecretsResult; -import org.apache.polaris.core.persistence.AtomicOperationMetaStoreManager; import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; import org.apache.polaris.core.storage.cache.StorageCredentialCache; import org.slf4j.Logger; @@ -68,13 +67,17 @@ public class JdbcMetaStoreManagerFactory implements MetaStoreManagerFactory { final Map storageCredentialCacheMap = new HashMap<>(); final Map entityCacheMap = new HashMap<>(); final Map> sessionSupplierMap = new HashMap<>(); + final Map dataSourceMap; protected final PolarisDiagnostics diagServices = new PolarisDefaultDiagServiceImpl(); // TODO: Pending discussion of if we should have one Database per realm or 1 schema per realm // or realm should be a primary key on all the tables. - @Inject Instance dataSource; @Inject PolarisStorageIntegrationProvider storageIntegrationProvider; + public static final String DEFAULT_DATA_SOURCE_NAME = ""; - protected JdbcMetaStoreManagerFactory() {} + @Inject + protected JdbcMetaStoreManagerFactory(Map dataSourceMap) { + this.dataSourceMap = dataSourceMap; + } protected PrincipalSecretsGenerator secretsGenerator( RealmContext realmContext, @Nullable RootCredentialsSet rootCredentialsSet) { @@ -92,7 +95,8 @@ protected PolarisMetaStoreManager createNewMetaStoreManager() { private void initializeForRealm( RealmContext realmContext, RootCredentialsSet rootCredentialsSet, boolean isBootstrap) { - DatasourceOperations databaseOperations = getDatasourceOperations(isBootstrap); + DatasourceOperations databaseOperations = + getDatasourceOperations(isBootstrap, realmContext.getRealmIdentifier()); sessionSupplierMap.put( realmContext.getRealmIdentifier(), () -> @@ -106,13 +110,18 @@ private void initializeForRealm( metaStoreManagerMap.put(realmContext.getRealmIdentifier(), metaStoreManager); } - private DatasourceOperations getDatasourceOperations(boolean isBootstrap) { - DatasourceOperations databaseOperations = new DatasourceOperations(dataSource.get()); + private DatasourceOperations getDatasourceOperations( + boolean isBootstrap, String realmIdentifier) { + // Idea is that worst case will do have it under default datasource name. + DatasourceOperations databaseOperations = + new DatasourceOperations( + dataSourceMap.getOrDefault( + realmIdentifier, dataSourceMap.get(DEFAULT_DATA_SOURCE_NAME))); if (isBootstrap) { // TODO: see if we need to take script from Quarkus or can we just // use the script committed in the repo. try { - databaseOperations.executeScript("postgres/schema-v1-postgresql.sql"); + databaseOperations.executeScript("postgres/schema-v1.sql"); } catch (SQLException e) { throw new RuntimeException( String.format("Error executing sql script: %s", e.getMessage()), e); diff --git a/extension/persistence/relational-jdbc/src/main/resources/h2/schema-v1-h2.sql b/extension/persistence/relational-jdbc/src/main/resources/h2/schema-v1.sql similarity index 100% rename from extension/persistence/relational-jdbc/src/main/resources/h2/schema-v1-h2.sql rename to extension/persistence/relational-jdbc/src/main/resources/h2/schema-v1.sql diff --git a/extension/persistence/relational-jdbc/src/main/resources/postgres/schema-v1-postgresql.sql b/extension/persistence/relational-jdbc/src/main/resources/postgres/schema-v1.sql similarity index 100% rename from extension/persistence/relational-jdbc/src/main/resources/postgres/schema-v1-postgresql.sql rename to extension/persistence/relational-jdbc/src/main/resources/postgres/schema-v1.sql diff --git a/extension/persistence/relational-jdbc/src/test/java/org/apache/polaris/extension/persistence/impl/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java b/extension/persistence/relational-jdbc/src/test/java/org/apache/polaris/extension/persistence/impl/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java index cdc31685b7..056f09d84a 100644 --- a/extension/persistence/relational-jdbc/src/test/java/org/apache/polaris/extension/persistence/impl/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java +++ b/extension/persistence/relational-jdbc/src/test/java/org/apache/polaris/extension/persistence/impl/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java @@ -49,7 +49,7 @@ protected PolarisTestMetaStoreManager createPolarisTestMetaStoreManager() { PolarisDiagnostics diagServices = new PolarisDefaultDiagServiceImpl(); DatasourceOperations datasourceOperations = new DatasourceOperations(createH2DataSource()); try { - datasourceOperations.executeScript("h2/schema-v1-h2.sql"); + datasourceOperations.executeScript("h2/schema-v1.sql"); } catch (SQLException e) { throw new RuntimeException(String.format("Error executing h2 script: %s", e.getMessage()), e); } diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java index 2ab3f9325d..e5c8619e1c 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java @@ -99,7 +99,9 @@ import org.apache.polaris.service.it.ext.PolarisIntegrationTestExtension; import org.apache.polaris.service.types.GenericTable; import org.assertj.core.api.Assertions; +import org.assertj.core.api.Assumptions; import org.assertj.core.api.InstanceOfAssertFactories; +import org.assertj.core.configuration.PreferredAssumptionException; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; @@ -178,6 +180,10 @@ static void setup( externalCatalogBase = testRootUri.resolve("external-catalog"); } + static { + Assumptions.setPreferredAssumptionException(PreferredAssumptionException.JUNIT5); + } + @AfterAll static void close() throws Exception { client.close(); diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java index 8dc48837bf..69215e4a88 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java @@ -37,9 +37,10 @@ import org.apache.polaris.service.it.env.PolarisApiEndpoints; import org.apache.polaris.service.it.env.PolarisClient; import org.apache.polaris.service.it.ext.PolarisIntegrationTestExtension; +import org.assertj.core.api.Assumptions; +import org.assertj.core.configuration.PreferredAssumptionException; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; @@ -57,6 +58,9 @@ */ @ExtendWith(PolarisIntegrationTestExtension.class) public abstract class PolarisRestCatalogViewIntegrationBase extends ViewCatalogTests { + static { + Assumptions.setPreferredAssumptionException(PreferredAssumptionException.JUNIT5); + } private static ClientCredentials adminCredentials; private static PolarisApiEndpoints endpoints; @@ -80,7 +84,7 @@ static void close() throws Exception { @BeforeEach public void before(TestInfo testInfo) { - Assumptions.assumeFalse(shouldSkip()); + org.junit.jupiter.api.Assumptions.assumeFalse(shouldSkip()); String principalName = client.newEntityName("snowman-rest"); String principalRoleName = client.newEntityName("rest-admin"); diff --git a/quarkus/admin/src/main/java/org/apache/polaris/admintool/config/QuarkusProducers.java b/quarkus/admin/src/main/java/org/apache/polaris/admintool/config/QuarkusProducers.java index 07ad023629..7f9b9ffd0b 100644 --- a/quarkus/admin/src/main/java/org/apache/polaris/admintool/config/QuarkusProducers.java +++ b/quarkus/admin/src/main/java/org/apache/polaris/admintool/config/QuarkusProducers.java @@ -18,6 +18,8 @@ */ package org.apache.polaris.admintool.config; +import io.quarkus.arc.All; +import io.quarkus.arc.InstanceHandle; import io.smallrye.common.annotation.Identifier; import jakarta.annotation.Nullable; import jakarta.enterprise.context.ApplicationScoped; @@ -25,6 +27,10 @@ import jakarta.enterprise.inject.Instance; import jakarta.enterprise.inject.Produces; import java.time.Clock; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import javax.sql.DataSource; import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.config.PolarisConfigurationStore; @@ -36,6 +42,8 @@ public class QuarkusProducers { + public static final String DEFAULT_DATA_SOURCE_NAME = ""; + @Produces public MetaStoreManagerFactory metaStoreManagerFactory( @ConfigProperty(name = "polaris.persistence.type") String persistenceType, @@ -76,4 +84,23 @@ public PolarisConfigurationStore configurationStore() { // A configuration store is not required when running the admin tool. return new PolarisConfigurationStore() {}; } + + @Produces + public Map dataSourceMap(@All List> dataSources) { + Map dataSourceMap = new HashMap<>(); + for (InstanceHandle handle : dataSources) { + String name = handle.getBean().getName(); + name = name == null ? DEFAULT_DATA_SOURCE_NAME : unquoteDataSourceName(name); + dataSourceMap.put(name, handle.get()); + } + + return dataSourceMap; + } + + private static String unquoteDataSourceName(String dataSourceName) { + if (dataSourceName.startsWith("\"") && dataSourceName.endsWith("\"")) { + dataSourceName = dataSourceName.substring(1, dataSourceName.length() - 1); + } + return dataSourceName; + } } diff --git a/quarkus/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcBootstrapCommandTest.java b/quarkus/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcBootstrapCommandTest.java index 9cf0ed3d92..2e45ba3f1e 100644 --- a/quarkus/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcBootstrapCommandTest.java +++ b/quarkus/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcBootstrapCommandTest.java @@ -16,7 +16,6 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.polaris.admintool.relational.jdbc; import io.quarkus.test.junit.TestProfile; diff --git a/quarkus/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcProfile.java b/quarkus/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcProfile.java index 652e4c4d2f..79b918940b 100644 --- a/quarkus/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcProfile.java +++ b/quarkus/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcProfile.java @@ -26,16 +26,16 @@ import org.apache.polaris.admintool.PostgresRelationalJdbcLifeCycleManagement; public class RelationalJdbcProfile implements QuarkusTestProfile { - @Override - public Map getConfigOverrides() { - return Map.of(); - } + @Override + public Map getConfigOverrides() { + return Map.of(); + } - @Override - public List testResources() { - return List.of( - new TestResourceEntry( - PostgresRelationalJdbcLifeCycleManagement.class, - Map.of(INIT_SCRIPT, "org/apache/polaris/admintool/init.sql"))); - } + @Override + public List testResources() { + return List.of( + new TestResourceEntry( + PostgresRelationalJdbcLifeCycleManagement.class, + Map.of(INIT_SCRIPT, "org/apache/polaris/admintool/init.sql"))); + } } diff --git a/quarkus/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcPurgeCommandTest.java b/quarkus/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcPurgeCommandTest.java index a8c123885f..ddd36738b6 100644 --- a/quarkus/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcPurgeCommandTest.java +++ b/quarkus/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcPurgeCommandTest.java @@ -16,7 +16,6 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.polaris.admintool.relational.jdbc; import io.quarkus.test.junit.TestProfile; @@ -26,13 +25,13 @@ @TestProfile(RelationalJdbcPurgeCommandTest.Profile.class) public class RelationalJdbcPurgeCommandTest extends PurgeCommandTestBase { - public static class Profile extends RelationalJdbcProfile { - @Override - public Map getConfigOverrides() { - return ImmutableMap.builder() - .putAll(super.getConfigOverrides()) - .put("pre-bootstrap", "true") - .build(); - } + public static class Profile extends RelationalJdbcProfile { + @Override + public Map getConfigOverrides() { + return ImmutableMap.builder() + .putAll(super.getConfigOverrides()) + .put("pre-bootstrap", "true") + .build(); } -} \ No newline at end of file + } +} diff --git a/quarkus/admin/src/testFixtures/java/org/apache/polaris/admintool/PostgresRelationalJdbcLifeCycleManagement.java b/quarkus/admin/src/testFixtures/java/org/apache/polaris/admintool/PostgresRelationalJdbcLifeCycleManagement.java index 6a983f3781..592bb73990 100644 --- a/quarkus/admin/src/testFixtures/java/org/apache/polaris/admintool/PostgresRelationalJdbcLifeCycleManagement.java +++ b/quarkus/admin/src/testFixtures/java/org/apache/polaris/admintool/PostgresRelationalJdbcLifeCycleManagement.java @@ -16,7 +16,6 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.polaris.admintool; import io.quarkus.test.common.DevServicesContext; @@ -26,61 +25,61 @@ import org.testcontainers.utility.DockerImageName; public class PostgresRelationalJdbcLifeCycleManagement - implements QuarkusTestResourceLifecycleManager, DevServicesContext.ContextAware { - private static final String INIT_SCRIPT = "init-script"; - - private PostgreSQLContainer postgres; - private String initScript; - private DevServicesContext context; + implements QuarkusTestResourceLifecycleManager, DevServicesContext.ContextAware { + private static final String INIT_SCRIPT = "init-script"; - @Override - public void init(Map initArgs) { - initScript = initArgs.get(INIT_SCRIPT); - } + private PostgreSQLContainer postgres; + private String initScript; + private DevServicesContext context; - @Override - @SuppressWarnings("resource") - public Map start() { - postgres = - new PostgreSQLContainer<>(DockerImageName.parse("postgres:17")) - .withDatabaseName("polaris_db") - .withUsername("polaris") - .withPassword("polaris"); + @Override + public void init(Map initArgs) { + initScript = initArgs.get(INIT_SCRIPT); + } - if (initScript != null) { - postgres.withInitScript(initScript); - } + @Override + @SuppressWarnings("resource") + public Map start() { + postgres = + new PostgreSQLContainer<>(DockerImageName.parse("postgres:17")) + .withDatabaseName("polaris_db") + .withUsername("polaris") + .withPassword("polaris"); - context.containerNetworkId().ifPresent(postgres::withNetworkMode); - postgres.start(); - return Map.of( - "polaris.persistence.type", - "relational-jdbc", - "quarkus.datasource.db-kind", - "pgsql", - "quarkus.datasource.jdbc.url", - postgres.getJdbcUrl(), - "quarkus.datasource.username", - postgres.getUsername(), - "quarkus.datasource.password", - postgres.getPassword(), - "quarkus.datasource.jdbc.initial-size", - "10"); + if (initScript != null) { + postgres.withInitScript(initScript); } - @Override - public void stop() { - if (postgres != null) { - try { - postgres.stop(); - } finally { - postgres = null; - } - } - } + context.containerNetworkId().ifPresent(postgres::withNetworkMode); + postgres.start(); + return Map.of( + "polaris.persistence.type", + "relational-jdbc", + "quarkus.datasource.db-kind", + "pgsql", + "quarkus.datasource.jdbc.url", + postgres.getJdbcUrl(), + "quarkus.datasource.username", + postgres.getUsername(), + "quarkus.datasource.password", + postgres.getPassword(), + "quarkus.datasource.jdbc.initial-size", + "10"); + } - @Override - public void setIntegrationTestContext(DevServicesContext context) { - this.context = context; + @Override + public void stop() { + if (postgres != null) { + try { + postgres.stop(); + } finally { + postgres = null; + } } -} \ No newline at end of file + } + + @Override + public void setIntegrationTestContext(DevServicesContext context) { + this.context = context; + } +} diff --git a/quarkus/server/build.gradle.kts b/quarkus/server/build.gradle.kts index 9e93b4cdb0..cf55d53751 100644 --- a/quarkus/server/build.gradle.kts +++ b/quarkus/server/build.gradle.kts @@ -51,7 +51,11 @@ dependencies { runtimeOnly(project(":polaris-eclipselink")) runtimeOnly("org.postgresql:postgresql") runtimeOnly(project(":polaris-relational-jdbc")) - implementation("io.quarkus:quarkus-jdbc-postgresql") + runtimeOnly("io.quarkus:quarkus-jdbc-postgresql") { + exclude(group = "org.antlr", module = "antlr4-runtime") + exclude(group = "org.scala-lang", module = "scala-library") + exclude(group = "org.scala-lang", module = "scala-reflect") + } // enforce the Quarkus _platform_ here, to get a consistent and validated set of dependencies implementation(enforcedPlatform(libs.quarkus.bom)) diff --git a/quarkus/service/build.gradle.kts b/quarkus/service/build.gradle.kts index f5fb7c5aed..a22de67579 100644 --- a/quarkus/service/build.gradle.kts +++ b/quarkus/service/build.gradle.kts @@ -23,11 +23,6 @@ plugins { id("polaris-quarkus") } -//configurations.all { -// // exclude junit4 dependency for this module -// exclude(group = "junit", module = "junit") -//} - dependencies { implementation(project(":polaris-core")) implementation(project(":polaris-api-management-service")) @@ -114,7 +109,11 @@ dependencies { testImplementation("software.amazon.awssdk:dynamodb") runtimeOnly(project(":polaris-relational-jdbc")) - implementation("io.quarkus:quarkus-jdbc-postgresql") + implementation("io.quarkus:quarkus-jdbc-postgresql") { + exclude(group = "org.antlr", module = "antlr4-runtime") + exclude(group = "org.scala-lang", module = "scala-library") + exclude(group = "org.scala-lang", module = "scala-reflect") + } testImplementation(platform(libs.quarkus.bom)) testImplementation("io.quarkus:quarkus-junit5") testImplementation("io.quarkus:quarkus-junit5-mockito") @@ -125,7 +124,11 @@ dependencies { testImplementation(libs.threeten.extra) testImplementation(libs.hawkular.agent.prometheus.scraper) - testFixturesApi(enforcedPlatform(libs.quarkus.bom)) + testFixturesApi(enforcedPlatform(libs.quarkus.bom)) { + exclude(group = "org.antlr", module = "antlr4-runtime") + exclude(group = "org.scala-lang", module = "scala-library") + exclude(group = "org.scala-lang", module = "scala-reflect") + } testFixturesApi("io.quarkus:quarkus-junit5") testFixturesApi(platform(libs.testcontainers.bom)) testFixturesApi("org.testcontainers:testcontainers") diff --git a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusApplicationIT.java b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusApplicationIT.java index 0bcf15b183..d8be531a8f 100644 --- a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusApplicationIT.java +++ b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusApplicationIT.java @@ -16,15 +16,12 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.polaris.service.quarkus.it.relational.jdbc; import io.quarkus.test.junit.QuarkusIntegrationTest; -import io.quarkus.test.junit.QuarkusTest; import io.quarkus.test.junit.TestProfile; import org.apache.polaris.service.it.test.PolarisApplicationIntegrationTest; @TestProfile(RelationalJdbcProfile.class) -@QuarkusTest -public class JdbcQuarkusApplicationIT extends PolarisApplicationIntegrationTest { -} +@QuarkusIntegrationTest +public class JdbcQuarkusApplicationIT extends PolarisApplicationIntegrationTest {} diff --git a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusManagementServiceIT.java b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusManagementServiceIT.java index 6e78f5d5fb..5fbe600325 100644 --- a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusManagementServiceIT.java +++ b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusManagementServiceIT.java @@ -16,14 +16,12 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.polaris.service.quarkus.it.relational.jdbc; -import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.QuarkusIntegrationTest; import io.quarkus.test.junit.TestProfile; import org.apache.polaris.service.it.test.PolarisManagementServiceIntegrationTest; @TestProfile(RelationalJdbcProfile.class) -@QuarkusTest -public class JdbcQuarkusManagementServiceIT extends PolarisManagementServiceIntegrationTest { -} +@QuarkusIntegrationTest +public class JdbcQuarkusManagementServiceIT extends PolarisManagementServiceIntegrationTest {} diff --git a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusRestCatalogViewAwsIT.java b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusRestCatalogViewAwsIT.java index 4fb79fad8f..958b4d0103 100644 --- a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusRestCatalogViewAwsIT.java +++ b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusRestCatalogViewAwsIT.java @@ -16,14 +16,12 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.polaris.service.quarkus.it.relational.jdbc; -import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.QuarkusIntegrationTest; import io.quarkus.test.junit.TestProfile; import org.apache.polaris.service.it.test.PolarisRestCatalogViewAwsIntegrationTest; @TestProfile(RelationalJdbcProfile.class) -@QuarkusTest -public class JdbcQuarkusRestCatalogViewAwsIT extends PolarisRestCatalogViewAwsIntegrationTest { -} +@QuarkusIntegrationTest +public class JdbcQuarkusRestCatalogViewAwsIT extends PolarisRestCatalogViewAwsIntegrationTest {} diff --git a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusRestCatalogViewAzureIT.java b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusRestCatalogViewAzureIT.java index a291e42097..64ef6bd6b9 100644 --- a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusRestCatalogViewAzureIT.java +++ b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusRestCatalogViewAzureIT.java @@ -16,14 +16,12 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.polaris.service.quarkus.it.relational.jdbc; -import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.QuarkusIntegrationTest; import io.quarkus.test.junit.TestProfile; import org.apache.polaris.service.it.test.PolarisRestCatalogViewAzureIntegrationTest; @TestProfile(RelationalJdbcProfile.class) -@QuarkusTest -public class JdbcQuarkusRestCatalogViewAzureIT extends PolarisRestCatalogViewAzureIntegrationTest { -} +@QuarkusIntegrationTest +public class JdbcQuarkusRestCatalogViewAzureIT extends PolarisRestCatalogViewAzureIntegrationTest {} diff --git a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusRestCatalogViewGcpIT.java b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusRestCatalogViewGcpIT.java index 0bd60eee9e..0b9fba8546 100644 --- a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusRestCatalogViewGcpIT.java +++ b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusRestCatalogViewGcpIT.java @@ -16,15 +16,12 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.polaris.service.quarkus.it.relational.jdbc; -import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.QuarkusIntegrationTest; import io.quarkus.test.junit.TestProfile; import org.apache.polaris.service.it.test.PolarisRestCatalogViewGcpIntegrationTest; -import org.apache.polaris.service.quarkus.it.QuarkusRestCatalogViewGcpIT; @TestProfile(RelationalJdbcProfile.class) -@QuarkusTest -public class JdbcQuarkusRestCatalogViewGcpIT extends PolarisRestCatalogViewGcpIntegrationTest { -} +@QuarkusIntegrationTest +public class JdbcQuarkusRestCatalogViewGcpIT extends PolarisRestCatalogViewGcpIntegrationTest {} diff --git a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusViewFileIT.java b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusViewFileIT.java index 2cf23a0a43..8b67272395 100644 --- a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusViewFileIT.java +++ b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusViewFileIT.java @@ -16,14 +16,12 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.polaris.service.quarkus.it.relational.jdbc; -import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.QuarkusIntegrationTest; import io.quarkus.test.junit.TestProfile; import org.apache.polaris.service.it.test.PolarisRestCatalogViewFileIntegrationTest; @TestProfile(RelationalJdbcProfile.class) -@QuarkusTest -public class JdbcQuarkusViewFileIT extends PolarisRestCatalogViewFileIntegrationTest { -} +@QuarkusIntegrationTest +public class JdbcQuarkusViewFileIT extends PolarisRestCatalogViewFileIntegrationTest {} diff --git a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcRestCatalogIT.java b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcRestCatalogIT.java index 9ebf0728eb..722abe2bfb 100644 --- a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcRestCatalogIT.java +++ b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcRestCatalogIT.java @@ -16,14 +16,12 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.polaris.service.quarkus.it.relational.jdbc; -import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.QuarkusIntegrationTest; import io.quarkus.test.junit.TestProfile; import org.apache.polaris.service.it.test.PolarisRestCatalogIntegrationTest; @TestProfile(RelationalJdbcProfile.class) -@QuarkusTest -public class JdbcRestCatalogIT extends PolarisRestCatalogIntegrationTest { -} +@QuarkusIntegrationTest +public class JdbcRestCatalogIT extends PolarisRestCatalogIntegrationTest {} diff --git a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/RelationalJdbcProfile.java b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/RelationalJdbcProfile.java index f54f4659ad..7d1087be87 100644 --- a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/RelationalJdbcProfile.java +++ b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/RelationalJdbcProfile.java @@ -16,27 +16,22 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.polaris.service.quarkus.it.relational.jdbc; import io.quarkus.test.junit.QuarkusTestProfile; -import org.apache.polaris.service.quarkus.it.PostgresRelationalJdbcLifeCycleManagement; - import java.util.List; import java.util.Map; +import org.apache.polaris.service.quarkus.it.PostgresRelationalJdbcLifeCycleManagement; public class RelationalJdbcProfile implements QuarkusTestProfile { - @Override - public Map getConfigOverrides() { - return Map.of( - "polaris.persistence.auto-bootstrap-types", "relational-jdbc" - ); - } + @Override + public Map getConfigOverrides() { + return Map.of("polaris.persistence.auto-bootstrap-types", "relational-jdbc"); + } - @Override - public List testResources() { - return List.of( - new TestResourceEntry( - PostgresRelationalJdbcLifeCycleManagement.class, Map.of())); - } + @Override + public List testResources() { + return List.of( + new TestResourceEntry(PostgresRelationalJdbcLifeCycleManagement.class, Map.of())); + } } diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java index d7bfa74020..a3306156d3 100644 --- a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java @@ -18,6 +18,8 @@ */ package org.apache.polaris.service.quarkus.config; +import io.quarkus.arc.All; +import io.quarkus.arc.InstanceHandle; import io.quarkus.runtime.StartupEvent; import io.smallrye.common.annotation.Identifier; import io.smallrye.context.SmallRyeManagedExecutor; @@ -32,7 +34,10 @@ import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.core.Context; import java.time.Clock; - +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import javax.sql.DataSource; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.PolarisDiagnostics; @@ -74,6 +79,9 @@ import org.eclipse.microprofile.context.ThreadContext; public class QuarkusProducers { + + public static final String DEFAULT_DATA_SOURCE_NAME = ""; + @Produces @ApplicationScoped // cannot be singleton because it is mocked in tests public Clock clock() { @@ -256,9 +264,28 @@ public ActiveRolesProvider activeRolesProvider( @ConfigProperty(name = "polaris.active-roles-provider.type") String persistenceType, @Any Instance activeRolesProviders) { return activeRolesProviders.select(Identifier.Literal.of(persistenceType)).get(); - } + } public void closeTaskExecutor(@Disposes @Identifier("task-executor") ManagedExecutor executor) { executor.close(); } + + @Produces + public Map dataSourceMap(@All List> dataSources) { + Map dataSourceMap = new HashMap<>(); + for (InstanceHandle handle : dataSources) { + String name = handle.getBean().getName(); + name = name == null ? DEFAULT_DATA_SOURCE_NAME : unquoteDataSourceName(name); + dataSourceMap.put(name, handle.get()); + } + + return dataSourceMap; + } + + private static String unquoteDataSourceName(String dataSourceName) { + if (dataSourceName.startsWith("\"") && dataSourceName.endsWith("\"")) { + dataSourceName = dataSourceName.substring(1, dataSourceName.length() - 1); + } + return dataSourceName; + } } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java index 210a792e2b..832ac32bdb 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java @@ -127,6 +127,7 @@ import org.apache.polaris.service.types.TableUpdateNotification; import org.assertj.core.api.Assertions; import org.assertj.core.api.ThrowableAssert.ThrowingCallable; +import org.assertj.core.configuration.PreferredAssumptionException; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.BeforeAll; @@ -146,6 +147,10 @@ @TestProfile(IcebergCatalogTest.Profile.class) public abstract class IcebergCatalogTest extends CatalogTests { + static { + org.assertj.core.api.Assumptions.setPreferredAssumptionException( + PreferredAssumptionException.JUNIT5); + } public static class Profile implements QuarkusTestProfile { diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java index 8d7708dd73..5123ea0e17 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java @@ -68,6 +68,8 @@ import org.apache.polaris.service.catalog.io.FileIOFactory; import org.apache.polaris.service.config.RealmEntityManagerFactory; import org.apache.polaris.service.storage.PolarisStorageIntegrationProviderImpl; +import org.assertj.core.api.Assumptions; +import org.assertj.core.configuration.PreferredAssumptionException; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@ -78,6 +80,9 @@ @QuarkusTest @TestProfile(IcebergCatalogViewTest.Profile.class) public class IcebergCatalogViewTest extends ViewCatalogTests { + static { + Assumptions.setPreferredAssumptionException(PreferredAssumptionException.JUNIT5); + } public static class Profile implements QuarkusTestProfile { diff --git a/quarkus/service/src/testFixtures/java/org/apache/polaris/service/quarkus/it/PostgresRelationalJdbcLifeCycleManagement.java b/quarkus/service/src/testFixtures/java/org/apache/polaris/service/quarkus/it/PostgresRelationalJdbcLifeCycleManagement.java index 1c3835e011..439749942b 100644 --- a/quarkus/service/src/testFixtures/java/org/apache/polaris/service/quarkus/it/PostgresRelationalJdbcLifeCycleManagement.java +++ b/quarkus/service/src/testFixtures/java/org/apache/polaris/service/quarkus/it/PostgresRelationalJdbcLifeCycleManagement.java @@ -16,7 +16,6 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.polaris.service.quarkus.it; import io.quarkus.test.common.DevServicesContext; @@ -26,61 +25,61 @@ import org.testcontainers.utility.DockerImageName; public class PostgresRelationalJdbcLifeCycleManagement - implements QuarkusTestResourceLifecycleManager, DevServicesContext.ContextAware { - private static final String INIT_SCRIPT = "init-script"; - - private PostgreSQLContainer postgres; - private String initScript; - private DevServicesContext context; + implements QuarkusTestResourceLifecycleManager, DevServicesContext.ContextAware { + private static final String INIT_SCRIPT = "init-script"; - @Override - public void init(Map initArgs) { - initScript = initArgs.get(INIT_SCRIPT); - } + private PostgreSQLContainer postgres; + private String initScript; + private DevServicesContext context; - @Override - @SuppressWarnings("resource") - public Map start() { - postgres = - new PostgreSQLContainer<>(DockerImageName.parse("postgres:17")) - .withDatabaseName("polaris_db") - .withUsername("polaris") - .withPassword("polaris"); + @Override + public void init(Map initArgs) { + initScript = initArgs.get(INIT_SCRIPT); + } - if (initScript != null) { - postgres.withInitScript(initScript); - } + @Override + @SuppressWarnings("resource") + public Map start() { + postgres = + new PostgreSQLContainer<>(DockerImageName.parse("postgres:17")) + .withDatabaseName("polaris_db") + .withUsername("polaris") + .withPassword("polaris"); - context.containerNetworkId().ifPresent(postgres::withNetworkMode); - postgres.start(); - return Map.of( - "polaris.persistence.type", - "relational-jdbc", - "quarkus.datasource.db-kind", - "pgsql", - "quarkus.datasource.jdbc.url", - postgres.getJdbcUrl(), - "quarkus.datasource.username", - postgres.getUsername(), - "quarkus.datasource.password", - postgres.getPassword(), - "quarkus.datasource.jdbc.initial-size", - "10"); + if (initScript != null) { + postgres.withInitScript(initScript); } - @Override - public void stop() { - if (postgres != null) { - try { - postgres.stop(); - } finally { - postgres = null; - } - } - } + context.containerNetworkId().ifPresent(postgres::withNetworkMode); + postgres.start(); + return Map.of( + "polaris.persistence.type", + "relational-jdbc", + "quarkus.datasource.db-kind", + "pgsql", + "quarkus.datasource.jdbc.url", + postgres.getJdbcUrl(), + "quarkus.datasource.username", + postgres.getUsername(), + "quarkus.datasource.password", + postgres.getPassword(), + "quarkus.datasource.jdbc.initial-size", + "10"); + } - @Override - public void setIntegrationTestContext(DevServicesContext context) { - this.context = context; + @Override + public void stop() { + if (postgres != null) { + try { + postgres.stop(); + } finally { + postgres = null; + } } + } + + @Override + public void setIntegrationTestContext(DevServicesContext context) { + this.context = context; + } } From 155f715bfb679dbe2bdbca44730e0562dc7fa88c Mon Sep 17 00:00:00 2001 From: Prashant Date: Thu, 24 Apr 2025 19:04:46 -0700 Subject: [PATCH 04/11] more review feedbacks --- bom/build.gradle.kts | 1 + .../relational/jdbc/DatabaseType.java | 22 +++-- .../relational/jdbc/DatasourceOperations.java | 19 ++++- .../jdbc/JdbcMetaStoreManagerFactory.java | 33 ++++--- ...anagerWithJdbcBasePersistenceImplTest.java | 9 +- gradle/projects.main.properties | 1 + quarkus/admin/build.gradle.kts | 1 + .../admintool/config/QuarkusProducers.java | 23 ++--- ...e.java => RelationalJdbcAdminProfile.java} | 6 +- .../RelationalJdbcBootstrapCommandTest.java | 2 +- .../jdbc/RelationalJdbcPurgeCommandTest.java | 2 +- quarkus/service/build.gradle.kts | 1 + .../jdbc/JdbcQuarkusApplicationIT.java | 1 + .../jdbc/JdbcQuarkusManagementServiceIT.java | 1 + .../JdbcQuarkusRestCatalogViewAzureIT.java | 27 ------ .../jdbc/JdbcQuarkusViewFileIT.java | 1 + .../it/relational/jdbc/JdbcRestCatalogIT.java | 1 + .../quarkus/config/QuarkusProducers.java | 23 ++--- ...gresRelationalJdbcLifeCycleManagement.java | 85 ------------------- .../build.gradle.kts} | 29 +++++-- ...gresRelationalJdbcLifeCycleManagement.java | 5 +- .../test/commons}/RelationalJdbcProfile.java | 6 +- 22 files changed, 104 insertions(+), 195 deletions(-) rename quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusRestCatalogViewGcpIT.java => extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/DatabaseType.java (66%) rename quarkus/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/{RelationalJdbcProfile.java => RelationalJdbcAdminProfile.java} (86%) delete mode 100644 quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusRestCatalogViewAzureIT.java delete mode 100644 quarkus/service/src/testFixtures/java/org/apache/polaris/service/quarkus/it/PostgresRelationalJdbcLifeCycleManagement.java rename quarkus/{service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusRestCatalogViewAwsIT.java => test-commons/build.gradle.kts} (55%) rename quarkus/{admin/src/testFixtures/java/org/apache/polaris/admintool => test-commons/src/main/java/org/apache/polaris/test/commons}/PostgresRelationalJdbcLifeCycleManagement.java (96%) rename quarkus/{service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc => test-commons/src/main/java/org/apache/polaris/test/commons}/RelationalJdbcProfile.java (83%) diff --git a/bom/build.gradle.kts b/bom/build.gradle.kts index 11859b32c9..4fc33d96c2 100644 --- a/bom/build.gradle.kts +++ b/bom/build.gradle.kts @@ -46,6 +46,7 @@ dependencies { api(project(":polaris-jpa-model")) api(project(":polaris-quarkus-admin")) + api(project(":polaris-quarkus-test-commons")) api(project(":polaris-quarkus-defaults")) api(project(":polaris-quarkus-server")) api(project(":polaris-quarkus-service")) diff --git a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusRestCatalogViewGcpIT.java b/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/DatabaseType.java similarity index 66% rename from quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusRestCatalogViewGcpIT.java rename to extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/DatabaseType.java index 0b9fba8546..2f3108488e 100644 --- a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusRestCatalogViewGcpIT.java +++ b/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/DatabaseType.java @@ -16,12 +16,20 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.quarkus.it.relational.jdbc; +package org.apache.polaris.extension.persistence.relational.jdbc; -import io.quarkus.test.junit.QuarkusIntegrationTest; -import io.quarkus.test.junit.TestProfile; -import org.apache.polaris.service.it.test.PolarisRestCatalogViewGcpIntegrationTest; +public enum DatabaseType { + POSTGRES("postgres"), + H2("h2"); -@TestProfile(RelationalJdbcProfile.class) -@QuarkusIntegrationTest -public class JdbcQuarkusRestCatalogViewGcpIT extends PolarisRestCatalogViewGcpIntegrationTest {} + private final String displayName; // Store the user-friendly name + + DatabaseType(String displayName) { + this.displayName = displayName; + } + + // Method to get the user-friendly display name + public String getDisplayName() { + return displayName; + } +} diff --git a/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/DatasourceOperations.java b/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/DatasourceOperations.java index 4884dde4ee..4944ed492e 100644 --- a/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/DatasourceOperations.java +++ b/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/DatasourceOperations.java @@ -30,16 +30,14 @@ import java.sql.Statement; import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.Objects; import java.util.function.Function; import java.util.function.Predicate; import javax.sql.DataSource; import org.apache.polaris.extension.persistence.relational.jdbc.models.Converter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class DatasourceOperations { - private static final Logger LOGGER = LoggerFactory.getLogger(DatasourceOperations.class); private static final String ALREADY_EXISTS_STATE_POSTGRES = "42P07"; private static final String CONSTRAINT_VIOLATION_SQL_CODE = "23505"; @@ -50,6 +48,21 @@ public DatasourceOperations(DataSource datasource) { this.datasource = datasource; } + @Nonnull + public DatabaseType detect() { + try (Connection conn = borrowConnection()) { + String productName = conn.getMetaData().getDatabaseProductName().toLowerCase(Locale.ROOT); + return switch (productName) { + case "h2" -> DatabaseType.H2; + case "postgresql" -> DatabaseType.POSTGRES; + default -> + throw new IllegalStateException("Unsupported DatabaseType: '" + productName + "'"); + }; + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + /** * Execute SQL script * diff --git a/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java b/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java index 04175fb2b3..9c51c0338b 100644 --- a/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java +++ b/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java @@ -24,6 +24,7 @@ import jakarta.inject.Inject; import java.sql.SQLException; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.function.Supplier; import javax.sql.DataSource; @@ -67,16 +68,14 @@ public class JdbcMetaStoreManagerFactory implements MetaStoreManagerFactory { final Map storageCredentialCacheMap = new HashMap<>(); final Map entityCacheMap = new HashMap<>(); final Map> sessionSupplierMap = new HashMap<>(); - final Map dataSourceMap; + final List dataSources; + protected final PolarisDiagnostics diagServices = new PolarisDefaultDiagServiceImpl(); - // TODO: Pending discussion of if we should have one Database per realm or 1 schema per realm - // or realm should be a primary key on all the tables. @Inject PolarisStorageIntegrationProvider storageIntegrationProvider; - public static final String DEFAULT_DATA_SOURCE_NAME = ""; @Inject - protected JdbcMetaStoreManagerFactory(Map dataSourceMap) { - this.dataSourceMap = dataSourceMap; + protected JdbcMetaStoreManagerFactory(List dataSources) { + this.dataSources = dataSources; } protected PrincipalSecretsGenerator secretsGenerator( @@ -95,8 +94,7 @@ protected PolarisMetaStoreManager createNewMetaStoreManager() { private void initializeForRealm( RealmContext realmContext, RootCredentialsSet rootCredentialsSet, boolean isBootstrap) { - DatasourceOperations databaseOperations = - getDatasourceOperations(isBootstrap, realmContext.getRealmIdentifier()); + DatasourceOperations databaseOperations = getDatasourceOperations(isBootstrap); sessionSupplierMap.put( realmContext.getRealmIdentifier(), () -> @@ -110,18 +108,17 @@ private void initializeForRealm( metaStoreManagerMap.put(realmContext.getRealmIdentifier(), metaStoreManager); } - private DatasourceOperations getDatasourceOperations( - boolean isBootstrap, String realmIdentifier) { - // Idea is that worst case will do have it under default datasource name. - DatasourceOperations databaseOperations = - new DatasourceOperations( - dataSourceMap.getOrDefault( - realmIdentifier, dataSourceMap.get(DEFAULT_DATA_SOURCE_NAME))); + private DatasourceOperations getDatasourceOperations(boolean isBootstrap) { + // TODO: remove when multiple dataSources are supported. + if (dataSources.size() > 1) { + throw new IllegalStateException("More than one dataSources configured"); + } + DatasourceOperations databaseOperations = new DatasourceOperations(dataSources.getFirst()); if (isBootstrap) { - // TODO: see if we need to take script from Quarkus or can we just - // use the script committed in the repo. try { - databaseOperations.executeScript("postgres/schema-v1.sql"); + DatabaseType databaseType = databaseOperations.detect(); + databaseOperations.executeScript( + String.format("%s/schema-v1.sql", databaseType.getDisplayName())); } catch (SQLException e) { throw new RuntimeException( String.format("Error executing sql script: %s", e.getMessage()), e); diff --git a/extension/persistence/relational-jdbc/src/test/java/org/apache/polaris/extension/persistence/impl/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java b/extension/persistence/relational-jdbc/src/test/java/org/apache/polaris/extension/persistence/impl/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java index 056f09d84a..72ef15913b 100644 --- a/extension/persistence/relational-jdbc/src/test/java/org/apache/polaris/extension/persistence/impl/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java +++ b/extension/persistence/relational-jdbc/src/test/java/org/apache/polaris/extension/persistence/impl/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java @@ -31,6 +31,7 @@ import org.apache.polaris.core.persistence.AtomicOperationMetaStoreManager; import org.apache.polaris.core.persistence.BasePolarisMetaStoreManagerTest; import org.apache.polaris.core.persistence.PolarisTestMetaStoreManager; +import org.apache.polaris.extension.persistence.relational.jdbc.DatabaseType; import org.apache.polaris.extension.persistence.relational.jdbc.DatasourceOperations; import org.apache.polaris.extension.persistence.relational.jdbc.JdbcBasePersistenceImpl; import org.h2.jdbcx.JdbcConnectionPool; @@ -49,9 +50,13 @@ protected PolarisTestMetaStoreManager createPolarisTestMetaStoreManager() { PolarisDiagnostics diagServices = new PolarisDefaultDiagServiceImpl(); DatasourceOperations datasourceOperations = new DatasourceOperations(createH2DataSource()); try { - datasourceOperations.executeScript("h2/schema-v1.sql"); + datasourceOperations.executeScript( + String.format("%s/schema-v1.sql", DatabaseType.H2.getDisplayName())); } catch (SQLException e) { - throw new RuntimeException(String.format("Error executing h2 script: %s", e.getMessage()), e); + throw new RuntimeException( + String.format( + "Error executing %s script: %s", DatabaseType.H2.getDisplayName(), e.getMessage()), + e); } JdbcBasePersistenceImpl basePersistence = diff --git a/gradle/projects.main.properties b/gradle/projects.main.properties index 8debf10559..6161eee47b 100644 --- a/gradle/projects.main.properties +++ b/gradle/projects.main.properties @@ -30,6 +30,7 @@ polaris-quarkus-service=quarkus/service polaris-quarkus-server=quarkus/server polaris-quarkus-spark-tests=quarkus/spark-tests polaris-quarkus-admin=quarkus/admin +polaris-quarkus-test-commons=quarkus/test-commons polaris-quarkus-run-script=quarkus/run-script polaris-eclipselink=extension/persistence/eclipselink polaris-jpa-model=extension/persistence/jpa-model diff --git a/quarkus/admin/build.gradle.kts b/quarkus/admin/build.gradle.kts index c7f281bc14..7b19a132a7 100644 --- a/quarkus/admin/build.gradle.kts +++ b/quarkus/admin/build.gradle.kts @@ -54,6 +54,7 @@ dependencies { implementation("org.jboss.slf4j:slf4j-jboss-logmanager") + testImplementation(project(":polaris-quarkus-test-commons")) testFixturesApi(project(":polaris-core")) testFixturesApi(enforcedPlatform(libs.quarkus.bom)) diff --git a/quarkus/admin/src/main/java/org/apache/polaris/admintool/config/QuarkusProducers.java b/quarkus/admin/src/main/java/org/apache/polaris/admintool/config/QuarkusProducers.java index 7f9b9ffd0b..69161419ee 100644 --- a/quarkus/admin/src/main/java/org/apache/polaris/admintool/config/QuarkusProducers.java +++ b/quarkus/admin/src/main/java/org/apache/polaris/admintool/config/QuarkusProducers.java @@ -27,9 +27,8 @@ import jakarta.enterprise.inject.Instance; import jakarta.enterprise.inject.Produces; import java.time.Clock; -import java.util.HashMap; +import java.util.ArrayList; import java.util.List; -import java.util.Map; import javax.sql.DataSource; import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.PolarisDiagnostics; @@ -42,8 +41,6 @@ public class QuarkusProducers { - public static final String DEFAULT_DATA_SOURCE_NAME = ""; - @Produces public MetaStoreManagerFactory metaStoreManagerFactory( @ConfigProperty(name = "polaris.persistence.type") String persistenceType, @@ -86,21 +83,11 @@ public PolarisConfigurationStore configurationStore() { } @Produces - public Map dataSourceMap(@All List> dataSources) { - Map dataSourceMap = new HashMap<>(); + public List dataSources(@All List> dataSources) { + List dataSourceList = new ArrayList<>(); for (InstanceHandle handle : dataSources) { - String name = handle.getBean().getName(); - name = name == null ? DEFAULT_DATA_SOURCE_NAME : unquoteDataSourceName(name); - dataSourceMap.put(name, handle.get()); - } - - return dataSourceMap; - } - - private static String unquoteDataSourceName(String dataSourceName) { - if (dataSourceName.startsWith("\"") && dataSourceName.endsWith("\"")) { - dataSourceName = dataSourceName.substring(1, dataSourceName.length() - 1); + dataSourceList.add(handle.get()); } - return dataSourceName; + return dataSourceList; } } diff --git a/quarkus/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcProfile.java b/quarkus/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcAdminProfile.java similarity index 86% rename from quarkus/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcProfile.java rename to quarkus/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcAdminProfile.java index 79b918940b..a64002983d 100644 --- a/quarkus/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcProfile.java +++ b/quarkus/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcAdminProfile.java @@ -20,12 +20,12 @@ import static org.apache.polaris.admintool.PostgresTestResourceLifecycleManager.INIT_SCRIPT; -import io.quarkus.test.junit.QuarkusTestProfile; import java.util.List; import java.util.Map; -import org.apache.polaris.admintool.PostgresRelationalJdbcLifeCycleManagement; +import org.apache.polaris.test.commons.PostgresRelationalJdbcLifeCycleManagement; +import org.apache.polaris.test.commons.RelationalJdbcProfile; -public class RelationalJdbcProfile implements QuarkusTestProfile { +public class RelationalJdbcAdminProfile extends RelationalJdbcProfile { @Override public Map getConfigOverrides() { return Map.of(); diff --git a/quarkus/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcBootstrapCommandTest.java b/quarkus/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcBootstrapCommandTest.java index 2e45ba3f1e..dcd5eb1408 100644 --- a/quarkus/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcBootstrapCommandTest.java +++ b/quarkus/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcBootstrapCommandTest.java @@ -21,5 +21,5 @@ import io.quarkus.test.junit.TestProfile; import org.apache.polaris.admintool.BootstrapCommandTestBase; -@TestProfile(RelationalJdbcProfile.class) +@TestProfile(RelationalJdbcAdminProfile.class) public class RelationalJdbcBootstrapCommandTest extends BootstrapCommandTestBase {} diff --git a/quarkus/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcPurgeCommandTest.java b/quarkus/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcPurgeCommandTest.java index ddd36738b6..43163f531b 100644 --- a/quarkus/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcPurgeCommandTest.java +++ b/quarkus/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcPurgeCommandTest.java @@ -25,7 +25,7 @@ @TestProfile(RelationalJdbcPurgeCommandTest.Profile.class) public class RelationalJdbcPurgeCommandTest extends PurgeCommandTestBase { - public static class Profile extends RelationalJdbcProfile { + public static class Profile extends RelationalJdbcAdminProfile { @Override public Map getConfigOverrides() { return ImmutableMap.builder() diff --git a/quarkus/service/build.gradle.kts b/quarkus/service/build.gradle.kts index a22de67579..7468a540f9 100644 --- a/quarkus/service/build.gradle.kts +++ b/quarkus/service/build.gradle.kts @@ -124,6 +124,7 @@ dependencies { testImplementation(libs.threeten.extra) testImplementation(libs.hawkular.agent.prometheus.scraper) + testImplementation(project(":polaris-quarkus-test-commons")) testFixturesApi(enforcedPlatform(libs.quarkus.bom)) { exclude(group = "org.antlr", module = "antlr4-runtime") exclude(group = "org.scala-lang", module = "scala-library") diff --git a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusApplicationIT.java b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusApplicationIT.java index d8be531a8f..dd718db732 100644 --- a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusApplicationIT.java +++ b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusApplicationIT.java @@ -21,6 +21,7 @@ import io.quarkus.test.junit.QuarkusIntegrationTest; import io.quarkus.test.junit.TestProfile; import org.apache.polaris.service.it.test.PolarisApplicationIntegrationTest; +import org.apache.polaris.test.commons.RelationalJdbcProfile; @TestProfile(RelationalJdbcProfile.class) @QuarkusIntegrationTest diff --git a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusManagementServiceIT.java b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusManagementServiceIT.java index 5fbe600325..0961889fd6 100644 --- a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusManagementServiceIT.java +++ b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusManagementServiceIT.java @@ -21,6 +21,7 @@ import io.quarkus.test.junit.QuarkusIntegrationTest; import io.quarkus.test.junit.TestProfile; import org.apache.polaris.service.it.test.PolarisManagementServiceIntegrationTest; +import org.apache.polaris.test.commons.RelationalJdbcProfile; @TestProfile(RelationalJdbcProfile.class) @QuarkusIntegrationTest diff --git a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusRestCatalogViewAzureIT.java b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusRestCatalogViewAzureIT.java deleted file mode 100644 index 64ef6bd6b9..0000000000 --- a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusRestCatalogViewAzureIT.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.quarkus.it.relational.jdbc; - -import io.quarkus.test.junit.QuarkusIntegrationTest; -import io.quarkus.test.junit.TestProfile; -import org.apache.polaris.service.it.test.PolarisRestCatalogViewAzureIntegrationTest; - -@TestProfile(RelationalJdbcProfile.class) -@QuarkusIntegrationTest -public class JdbcQuarkusRestCatalogViewAzureIT extends PolarisRestCatalogViewAzureIntegrationTest {} diff --git a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusViewFileIT.java b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusViewFileIT.java index 8b67272395..7471032a52 100644 --- a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusViewFileIT.java +++ b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusViewFileIT.java @@ -21,6 +21,7 @@ import io.quarkus.test.junit.QuarkusIntegrationTest; import io.quarkus.test.junit.TestProfile; import org.apache.polaris.service.it.test.PolarisRestCatalogViewFileIntegrationTest; +import org.apache.polaris.test.commons.RelationalJdbcProfile; @TestProfile(RelationalJdbcProfile.class) @QuarkusIntegrationTest diff --git a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcRestCatalogIT.java b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcRestCatalogIT.java index 722abe2bfb..59b35fe912 100644 --- a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcRestCatalogIT.java +++ b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcRestCatalogIT.java @@ -21,6 +21,7 @@ import io.quarkus.test.junit.QuarkusIntegrationTest; import io.quarkus.test.junit.TestProfile; import org.apache.polaris.service.it.test.PolarisRestCatalogIntegrationTest; +import org.apache.polaris.test.commons.RelationalJdbcProfile; @TestProfile(RelationalJdbcProfile.class) @QuarkusIntegrationTest diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java index a3306156d3..cbede0a1e9 100644 --- a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java @@ -34,9 +34,8 @@ import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.core.Context; import java.time.Clock; -import java.util.HashMap; +import java.util.ArrayList; import java.util.List; -import java.util.Map; import javax.sql.DataSource; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; @@ -80,8 +79,6 @@ public class QuarkusProducers { - public static final String DEFAULT_DATA_SOURCE_NAME = ""; - @Produces @ApplicationScoped // cannot be singleton because it is mocked in tests public Clock clock() { @@ -271,21 +268,11 @@ public void closeTaskExecutor(@Disposes @Identifier("task-executor") ManagedExec } @Produces - public Map dataSourceMap(@All List> dataSources) { - Map dataSourceMap = new HashMap<>(); + public List dataSources(@All List> dataSources) { + List dataSourceList = new ArrayList<>(); for (InstanceHandle handle : dataSources) { - String name = handle.getBean().getName(); - name = name == null ? DEFAULT_DATA_SOURCE_NAME : unquoteDataSourceName(name); - dataSourceMap.put(name, handle.get()); - } - - return dataSourceMap; - } - - private static String unquoteDataSourceName(String dataSourceName) { - if (dataSourceName.startsWith("\"") && dataSourceName.endsWith("\"")) { - dataSourceName = dataSourceName.substring(1, dataSourceName.length() - 1); + dataSourceList.add(handle.get()); } - return dataSourceName; + return dataSourceList; } } diff --git a/quarkus/service/src/testFixtures/java/org/apache/polaris/service/quarkus/it/PostgresRelationalJdbcLifeCycleManagement.java b/quarkus/service/src/testFixtures/java/org/apache/polaris/service/quarkus/it/PostgresRelationalJdbcLifeCycleManagement.java deleted file mode 100644 index 439749942b..0000000000 --- a/quarkus/service/src/testFixtures/java/org/apache/polaris/service/quarkus/it/PostgresRelationalJdbcLifeCycleManagement.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.quarkus.it; - -import io.quarkus.test.common.DevServicesContext; -import io.quarkus.test.common.QuarkusTestResourceLifecycleManager; -import java.util.Map; -import org.testcontainers.containers.PostgreSQLContainer; -import org.testcontainers.utility.DockerImageName; - -public class PostgresRelationalJdbcLifeCycleManagement - implements QuarkusTestResourceLifecycleManager, DevServicesContext.ContextAware { - private static final String INIT_SCRIPT = "init-script"; - - private PostgreSQLContainer postgres; - private String initScript; - private DevServicesContext context; - - @Override - public void init(Map initArgs) { - initScript = initArgs.get(INIT_SCRIPT); - } - - @Override - @SuppressWarnings("resource") - public Map start() { - postgres = - new PostgreSQLContainer<>(DockerImageName.parse("postgres:17")) - .withDatabaseName("polaris_db") - .withUsername("polaris") - .withPassword("polaris"); - - if (initScript != null) { - postgres.withInitScript(initScript); - } - - context.containerNetworkId().ifPresent(postgres::withNetworkMode); - postgres.start(); - return Map.of( - "polaris.persistence.type", - "relational-jdbc", - "quarkus.datasource.db-kind", - "pgsql", - "quarkus.datasource.jdbc.url", - postgres.getJdbcUrl(), - "quarkus.datasource.username", - postgres.getUsername(), - "quarkus.datasource.password", - postgres.getPassword(), - "quarkus.datasource.jdbc.initial-size", - "10"); - } - - @Override - public void stop() { - if (postgres != null) { - try { - postgres.stop(); - } finally { - postgres = null; - } - } - } - - @Override - public void setIntegrationTestContext(DevServicesContext context) { - this.context = context; - } -} diff --git a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusRestCatalogViewAwsIT.java b/quarkus/test-commons/build.gradle.kts similarity index 55% rename from quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusRestCatalogViewAwsIT.java rename to quarkus/test-commons/build.gradle.kts index 958b4d0103..b3813c90be 100644 --- a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/JdbcQuarkusRestCatalogViewAwsIT.java +++ b/quarkus/test-commons/build.gradle.kts @@ -16,12 +16,27 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.quarkus.it.relational.jdbc; -import io.quarkus.test.junit.QuarkusIntegrationTest; -import io.quarkus.test.junit.TestProfile; -import org.apache.polaris.service.it.test.PolarisRestCatalogViewAwsIntegrationTest; +plugins { + alias(libs.plugins.jandex) + id("java-test-fixtures") +} -@TestProfile(RelationalJdbcProfile.class) -@QuarkusIntegrationTest -public class JdbcQuarkusRestCatalogViewAwsIT extends PolarisRestCatalogViewAwsIntegrationTest {} +configurations.all { + exclude(group = "org.antlr", module = "antlr4-runtime") + exclude(group = "org.scala-lang", module = "scala-library") + exclude(group = "org.scala-lang", module = "scala-reflect") +} + +java { + sourceCompatibility = JavaVersion.VERSION_21 + targetCompatibility = JavaVersion.VERSION_21 +} + +dependencies { + implementation(enforcedPlatform(libs.quarkus.bom)) + implementation("io.quarkus:quarkus-junit5") + implementation(platform(libs.testcontainers.bom)) + implementation("org.testcontainers:testcontainers") + implementation("org.testcontainers:postgresql") +} diff --git a/quarkus/admin/src/testFixtures/java/org/apache/polaris/admintool/PostgresRelationalJdbcLifeCycleManagement.java b/quarkus/test-commons/src/main/java/org/apache/polaris/test/commons/PostgresRelationalJdbcLifeCycleManagement.java similarity index 96% rename from quarkus/admin/src/testFixtures/java/org/apache/polaris/admintool/PostgresRelationalJdbcLifeCycleManagement.java rename to quarkus/test-commons/src/main/java/org/apache/polaris/test/commons/PostgresRelationalJdbcLifeCycleManagement.java index 592bb73990..a85a70909e 100644 --- a/quarkus/admin/src/testFixtures/java/org/apache/polaris/admintool/PostgresRelationalJdbcLifeCycleManagement.java +++ b/quarkus/test-commons/src/main/java/org/apache/polaris/test/commons/PostgresRelationalJdbcLifeCycleManagement.java @@ -16,7 +16,8 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.admintool; + +package org.apache.polaris.test.commons; import io.quarkus.test.common.DevServicesContext; import io.quarkus.test.common.QuarkusTestResourceLifecycleManager; @@ -26,7 +27,7 @@ public class PostgresRelationalJdbcLifeCycleManagement implements QuarkusTestResourceLifecycleManager, DevServicesContext.ContextAware { - private static final String INIT_SCRIPT = "init-script"; + public static final String INIT_SCRIPT = "init-script"; private PostgreSQLContainer postgres; private String initScript; diff --git a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/RelationalJdbcProfile.java b/quarkus/test-commons/src/main/java/org/apache/polaris/test/commons/RelationalJdbcProfile.java similarity index 83% rename from quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/RelationalJdbcProfile.java rename to quarkus/test-commons/src/main/java/org/apache/polaris/test/commons/RelationalJdbcProfile.java index 7d1087be87..38f7bd61a0 100644 --- a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/relational/jdbc/RelationalJdbcProfile.java +++ b/quarkus/test-commons/src/main/java/org/apache/polaris/test/commons/RelationalJdbcProfile.java @@ -16,12 +16,11 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.quarkus.it.relational.jdbc; +package org.apache.polaris.test.commons; import io.quarkus.test.junit.QuarkusTestProfile; import java.util.List; import java.util.Map; -import org.apache.polaris.service.quarkus.it.PostgresRelationalJdbcLifeCycleManagement; public class RelationalJdbcProfile implements QuarkusTestProfile { @Override @@ -32,6 +31,7 @@ public Map getConfigOverrides() { @Override public List testResources() { return List.of( - new TestResourceEntry(PostgresRelationalJdbcLifeCycleManagement.class, Map.of())); + new QuarkusTestProfile.TestResourceEntry( + PostgresRelationalJdbcLifeCycleManagement.class, Map.of())); } } From 99928c1355d465154d5b618c089cf295ced827c0 Mon Sep 17 00:00:00 2001 From: Prashant Date: Fri, 25 Apr 2025 09:05:54 -0700 Subject: [PATCH 05/11] Address yufei feedback --- .../relational/jdbc/DatabaseType.java | 8 + .../relational/jdbc/DatasourceOperations.java | 21 --- .../jdbc/JdbcBasePersistenceImpl.java | 154 +++++++----------- .../jdbc/JdbcMetaStoreManagerFactory.java | 9 +- quarkus/server/build.gradle.kts | 6 - quarkus/service/build.gradle.kts | 2 +- 6 files changed, 72 insertions(+), 128 deletions(-) diff --git a/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/DatabaseType.java b/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/DatabaseType.java index 2f3108488e..15e5f00fa0 100644 --- a/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/DatabaseType.java +++ b/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/DatabaseType.java @@ -32,4 +32,12 @@ public enum DatabaseType { public String getDisplayName() { return displayName; } + + public static DatabaseType fromDisplayName(String displayName) { + return switch (displayName) { + case "h2" -> DatabaseType.H2; + case "postgresql" -> DatabaseType.POSTGRES; + default -> throw new IllegalStateException("Unsupported DatabaseType: '" + displayName + "'"); + }; + } } diff --git a/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/DatasourceOperations.java b/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/DatasourceOperations.java index 4944ed492e..6fec8e67fe 100644 --- a/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/DatasourceOperations.java +++ b/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/DatasourceOperations.java @@ -30,7 +30,6 @@ import java.sql.Statement; import java.util.ArrayList; import java.util.List; -import java.util.Locale; import java.util.Objects; import java.util.function.Function; import java.util.function.Predicate; @@ -39,7 +38,6 @@ public class DatasourceOperations { - private static final String ALREADY_EXISTS_STATE_POSTGRES = "42P07"; private static final String CONSTRAINT_VIOLATION_SQL_CODE = "23505"; private final DataSource datasource; @@ -48,21 +46,6 @@ public DatasourceOperations(DataSource datasource) { this.datasource = datasource; } - @Nonnull - public DatabaseType detect() { - try (Connection conn = borrowConnection()) { - String productName = conn.getMetaData().getDatabaseProductName().toLowerCase(Locale.ROOT); - return switch (productName) { - case "h2" -> DatabaseType.H2; - case "postgresql" -> DatabaseType.POSTGRES; - default -> - throw new IllegalStateException("Unsupported DatabaseType: '" + productName + "'"); - }; - } catch (SQLException e) { - throw new RuntimeException(e); - } - } - /** * Execute SQL script * @@ -202,10 +185,6 @@ public boolean isConstraintViolation(SQLException e) { return CONSTRAINT_VIOLATION_SQL_CODE.equals(e.getSQLState()); } - public boolean isAlreadyExistsException(SQLException e) { - return ALREADY_EXISTS_STATE_POSTGRES.equals(e.getSQLState()); - } - private Connection borrowConnection() throws SQLException { return datasource.getConnection(); } diff --git a/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcBasePersistenceImpl.java b/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcBasePersistenceImpl.java index dcbbaff8bb..877ac62dd4 100644 --- a/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcBasePersistenceImpl.java +++ b/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcBasePersistenceImpl.java @@ -23,6 +23,7 @@ import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; import java.sql.SQLException; +import java.sql.Statement; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -86,12 +87,54 @@ public void writeEntity( @Nonnull PolarisBaseEntity entity, boolean nameOrParentChanged, PolarisBaseEntity originalEntity) { + persistEntity(callCtx, entity, originalEntity, datasourceOperations); + } + + @Override + public void writeEntities( + @Nonnull PolarisCallContext callCtx, + @Nonnull List entities, + List originalEntities) { + try { + datasourceOperations.runWithinTransaction( + statement -> { + for (int i = 0; i < entities.size(); i++) { + PolarisBaseEntity entity = entities.get(i); + PolarisBaseEntity originalEntity = + originalEntities != null ? originalEntities.get(i) : null; + + // first, check if the entity has already been created, in which case we will simply + // return it. + PolarisBaseEntity entityFound = + lookupEntity( + callCtx, entity.getCatalogId(), entity.getId(), entity.getTypeCode()); + if (entityFound != null && originalEntity == null) { + // probably the client retried, simply return it + // TODO: Check correctness of returning entityFound vs entity here. It may have + // already been updated after the creation. + continue; + } + persistEntity(callCtx, entity, originalEntity, statement); + } + return true; + }); + } catch (SQLException e) { + throw new RuntimeException( + String.format( + "Error executing the transaction for writing entities due to %s", e.getMessage()), + e); + } + } + + private void persistEntity( + @Nonnull PolarisCallContext callCtx, + @Nonnull PolarisBaseEntity entity, + PolarisBaseEntity originalEntity, + Object executor) { ModelEntity modelEntity = ModelEntity.fromEntity(entity); - String query; if (originalEntity == null) { try { - query = generateInsertQuery(modelEntity, realmId); - datasourceOperations.executeUpdate(query); + execute(executor, generateInsertQuery(modelEntity, realmId)); } catch (SQLException e) { if (datasourceOperations.isConstraintViolation(e)) { PolarisBaseEntity existingEntity = @@ -102,8 +145,6 @@ public void writeEntity( entity.getTypeCode(), entity.getName()); throw new EntityAlreadyExistsException(existingEntity, e); - } else if (datasourceOperations.isAlreadyExistsException(e)) { - throw new EntityAlreadyExistsException(entity, e); } else { throw new RuntimeException( String.format("Failed to write entity due to %s", e.getMessage()), e); @@ -120,13 +161,12 @@ public void writeEntity( originalEntity.getEntityVersion(), "realm_id", realmId); - query = generateUpdateQuery(modelEntity, params); try { - int rowsUpdated = datasourceOperations.executeUpdate(query); + int rowsUpdated = execute(executor, generateUpdateQuery(modelEntity, params)); if (rowsUpdated == 0) { throw new RetryOnConcurrencyException( "Entity '%s' id '%s' concurrently modified; expected version %s", - entity.getName(), entity.getId(), originalEntity.getEntityVersion()); + originalEntity.getName(), originalEntity.getId(), originalEntity.getEntityVersion()); } } catch (SQLException e) { throw new RuntimeException( @@ -135,97 +175,13 @@ public void writeEntity( } } - @Override - public void writeEntities( - @Nonnull PolarisCallContext callCtx, - @Nonnull List entities, - List originalEntities) { - try { - datasourceOperations.runWithinTransaction( - statement -> { - for (int i = 0; i < entities.size(); i++) { - PolarisBaseEntity entity = entities.get(i); - ModelEntity modelEntity = ModelEntity.fromEntity(entity); - boolean isUpdate = originalEntities != null && originalEntities.get(i) != null; - - // first, check if the entity has already been created, in which case we will simply - // return it. - PolarisBaseEntity entityFound = - lookupEntity( - callCtx, entity.getCatalogId(), entity.getId(), entity.getTypeCode()); - if (entityFound != null && !isUpdate) { - // probably the client retried, simply return it - // TODO: Check correctness of returning entityFound vs entity here. It may have - // already been updated after the creation. - continue; - } - // lookup by name - EntityNameLookupRecord exists = - lookupEntityIdAndSubTypeByName( - callCtx, - entity.getCatalogId(), - entity.getParentId(), - entity.getTypeCode(), - entity.getName()); - if (exists != null && !isUpdate) { - throw new EntityAlreadyExistsException(entity); - } - String query; - if (!isUpdate) { - try { - query = generateInsertQuery(modelEntity, realmId); - statement.executeUpdate(query); - } catch (SQLException e) { - if (datasourceOperations.isConstraintViolation(e)) { - PolarisBaseEntity existingEntity = - lookupEntityByName( - callCtx, - entity.getCatalogId(), - entity.getParentId(), - entity.getTypeCode(), - entity.getName()); - throw new EntityAlreadyExistsException(existingEntity, e); - } else if (datasourceOperations.isAlreadyExistsException(e)) { - throw new EntityAlreadyExistsException(entity, e); - } else { - throw new RuntimeException( - String.format("Failed to write entity due to %s", e.getMessage()), e); - } - } - } else { - Map params = - Map.of( - "id", - originalEntities.get(i).getId(), - "catalog_id", - originalEntities.get(i).getCatalogId(), - "entity_version", - originalEntities.get(i).getEntityVersion(), - "realm_id", - realmId); - query = generateUpdateQuery(modelEntity, params); - try { - int rowsUpdated = statement.executeUpdate(query); - if (rowsUpdated == 0) { - throw new RetryOnConcurrencyException( - "Entity '%s' id '%s' concurrently modified; expected version %s", - entity.getName(), - entity.getId(), - originalEntities.get(i).getEntityVersion()); - } - } catch (SQLException e) { - throw new RuntimeException( - String.format("Failed to write entity due to %s", e.getMessage()), e); - } - } - } - return true; - }); - } catch (SQLException e) { - throw new RuntimeException( - String.format( - "Error executing the transaction for writing entities due to %s", e.getMessage()), - e); + private int execute(Object executor, String query) throws SQLException { + if (executor instanceof Statement) { + return ((Statement) executor).executeUpdate(query); + } else if (executor instanceof DatasourceOperations) { + return ((DatasourceOperations) executor).executeUpdate(query); + } else { + throw new IllegalArgumentException("Unsupported executor: " + executor); } } diff --git a/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java b/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java index 9c51c0338b..b33878a1e5 100644 --- a/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java +++ b/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java @@ -22,9 +22,11 @@ import jakarta.annotation.Nullable; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; +import java.sql.Connection; import java.sql.SQLException; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.function.Supplier; import javax.sql.DataSource; @@ -116,7 +118,12 @@ private DatasourceOperations getDatasourceOperations(boolean isBootstrap) { DatasourceOperations databaseOperations = new DatasourceOperations(dataSources.getFirst()); if (isBootstrap) { try { - DatabaseType databaseType = databaseOperations.detect(); + DatabaseType databaseType; + try (Connection connection = dataSources.getFirst().getConnection()) { + String productName = + connection.getMetaData().getDatabaseProductName().toLowerCase(Locale.ROOT); + databaseType = DatabaseType.fromDisplayName(productName); + } databaseOperations.executeScript( String.format("%s/schema-v1.sql", databaseType.getDisplayName())); } catch (SQLException e) { diff --git a/quarkus/server/build.gradle.kts b/quarkus/server/build.gradle.kts index cf55d53751..61c1b46660 100644 --- a/quarkus/server/build.gradle.kts +++ b/quarkus/server/build.gradle.kts @@ -50,12 +50,6 @@ dependencies { runtimeOnly(project(":polaris-eclipselink")) runtimeOnly("org.postgresql:postgresql") - runtimeOnly(project(":polaris-relational-jdbc")) - runtimeOnly("io.quarkus:quarkus-jdbc-postgresql") { - exclude(group = "org.antlr", module = "antlr4-runtime") - exclude(group = "org.scala-lang", module = "scala-library") - exclude(group = "org.scala-lang", module = "scala-reflect") - } // enforce the Quarkus _platform_ here, to get a consistent and validated set of dependencies implementation(enforcedPlatform(libs.quarkus.bom)) diff --git a/quarkus/service/build.gradle.kts b/quarkus/service/build.gradle.kts index 7468a540f9..ba2ffa6eb2 100644 --- a/quarkus/service/build.gradle.kts +++ b/quarkus/service/build.gradle.kts @@ -109,7 +109,7 @@ dependencies { testImplementation("software.amazon.awssdk:dynamodb") runtimeOnly(project(":polaris-relational-jdbc")) - implementation("io.quarkus:quarkus-jdbc-postgresql") { + runtimeOnly("io.quarkus:quarkus-jdbc-postgresql") { exclude(group = "org.antlr", module = "antlr4-runtime") exclude(group = "org.scala-lang", module = "scala-library") exclude(group = "org.scala-lang", module = "scala-reflect") From 41291f419254e2a94bf83ccc914e8eb5a3ed5a51 Mon Sep 17 00:00:00 2001 From: Prashant Date: Fri, 25 Apr 2025 14:06:08 -0700 Subject: [PATCH 06/11] Address feedbacks --- .../persistence/relational/jdbc/DatabaseType.java | 8 +++++++- .../relational/jdbc/JdbcMetaStoreManagerFactory.java | 9 +++------ .../persistence/relational/jdbc/models/ModelEntity.java | 7 +++++-- .../it/test/PolarisRestCatalogViewIntegrationBase.java | 2 +- 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/DatabaseType.java b/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/DatabaseType.java index 15e5f00fa0..cdf6ddc43d 100644 --- a/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/DatabaseType.java +++ b/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/DatabaseType.java @@ -18,6 +18,8 @@ */ package org.apache.polaris.extension.persistence.relational.jdbc; +import java.util.Locale; + public enum DatabaseType { POSTGRES("postgres"), H2("h2"); @@ -34,10 +36,14 @@ public String getDisplayName() { } public static DatabaseType fromDisplayName(String displayName) { - return switch (displayName) { + return switch (displayName.toLowerCase(Locale.ROOT)) { case "h2" -> DatabaseType.H2; case "postgresql" -> DatabaseType.POSTGRES; default -> throw new IllegalStateException("Unsupported DatabaseType: '" + displayName + "'"); }; } + + public String getInitScriptResource() { + return String.format("%s/schema-v1.sql", this.getDisplayName()); + } } diff --git a/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java b/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java index b33878a1e5..2aaafd5728 100644 --- a/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java +++ b/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java @@ -18,6 +18,7 @@ */ package org.apache.polaris.extension.persistence.relational.jdbc; +import com.google.common.base.Preconditions; import io.smallrye.common.annotation.Identifier; import jakarta.annotation.Nullable; import jakarta.enterprise.context.ApplicationScoped; @@ -26,7 +27,6 @@ import java.sql.SQLException; import java.util.HashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.function.Supplier; import javax.sql.DataSource; @@ -112,16 +112,13 @@ private void initializeForRealm( private DatasourceOperations getDatasourceOperations(boolean isBootstrap) { // TODO: remove when multiple dataSources are supported. - if (dataSources.size() > 1) { - throw new IllegalStateException("More than one dataSources configured"); - } + Preconditions.checkState(dataSources.size() == 1, "More than one dataSources configured"); DatasourceOperations databaseOperations = new DatasourceOperations(dataSources.getFirst()); if (isBootstrap) { try { DatabaseType databaseType; try (Connection connection = dataSources.getFirst().getConnection()) { - String productName = - connection.getMetaData().getDatabaseProductName().toLowerCase(Locale.ROOT); + String productName = connection.getMetaData().getDatabaseProductName(); databaseType = DatabaseType.fromDisplayName(productName); } databaseOperations.executeScript( diff --git a/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/models/ModelEntity.java b/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/models/ModelEntity.java index 92f6e35c77..948f9a8693 100644 --- a/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/models/ModelEntity.java +++ b/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/models/ModelEntity.java @@ -152,8 +152,11 @@ public ModelEntity fromResultSet(ResultSet r) throws SQLException { .purgeTimestamp(r.getObject("purge_timestamp", Long.class)) .toPurgeTimestamp(r.getObject("to_purge_timestamp", Long.class)) .lastUpdateTimestamp(r.getObject("last_update_timestamp", Long.class)) - .properties(r.getString("properties")) - .internalProperties(r.getString("internal_properties")) + .properties( + r.getString("properties")) // required for extracting when the underlying type is JSONB + .internalProperties( + r.getString( + "internal_properties")) // required for extracting when the underlying type is JSONB .grantRecordsVersion(r.getObject("grant_records_version", Integer.class)) .build(); } diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java index 69215e4a88..c5df1c5d8a 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java @@ -84,7 +84,7 @@ static void close() throws Exception { @BeforeEach public void before(TestInfo testInfo) { - org.junit.jupiter.api.Assumptions.assumeFalse(shouldSkip()); + Assumptions.assumeThat(shouldSkip()).isFalse(); String principalName = client.newEntityName("snowman-rest"); String principalRoleName = client.newEntityName("rest-admin"); From 4a2bf40a55d01249996451eb78f856f1125b13f2 Mon Sep 17 00:00:00 2001 From: Prashant Date: Fri, 25 Apr 2025 16:17:51 -0700 Subject: [PATCH 07/11] change the test impl --- quarkus/service/build.gradle.kts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/quarkus/service/build.gradle.kts b/quarkus/service/build.gradle.kts index ba2ffa6eb2..d2375b8ef6 100644 --- a/quarkus/service/build.gradle.kts +++ b/quarkus/service/build.gradle.kts @@ -130,11 +130,11 @@ dependencies { exclude(group = "org.scala-lang", module = "scala-library") exclude(group = "org.scala-lang", module = "scala-reflect") } - testFixturesApi("io.quarkus:quarkus-junit5") - testFixturesApi(platform(libs.testcontainers.bom)) - testFixturesApi("org.testcontainers:testcontainers") - testFixturesApi("org.testcontainers:postgresql") - testFixturesApi("org.postgresql:postgresql") + testImplementation("io.quarkus:quarkus-junit5") + testImplementation(platform(libs.testcontainers.bom)) + testImplementation("org.testcontainers:testcontainers") + testImplementation("org.testcontainers:postgresql") + testImplementation("org.postgresql:postgresql") } tasks.withType(Test::class.java).configureEach { From e28c052860e10feb667a7656319e1aa47f319382 Mon Sep 17 00:00:00 2001 From: Prashant Date: Fri, 25 Apr 2025 17:31:42 -0700 Subject: [PATCH 08/11] Address feedbacks --- .../jdbc/JdbcMetaStoreManagerFactory.java | 13 +++++-------- .../polaris/admintool/config/QuarkusProducers.java | 14 -------------- quarkus/server/build.gradle.kts | 6 ++++++ quarkus/service/build.gradle.kts | 5 ----- .../service/quarkus/config/QuarkusProducers.java | 14 -------------- 5 files changed, 11 insertions(+), 41 deletions(-) diff --git a/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java b/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java index 2aaafd5728..23860207d2 100644 --- a/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java +++ b/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java @@ -18,7 +18,6 @@ */ package org.apache.polaris.extension.persistence.relational.jdbc; -import com.google.common.base.Preconditions; import io.smallrye.common.annotation.Identifier; import jakarta.annotation.Nullable; import jakarta.enterprise.context.ApplicationScoped; @@ -26,7 +25,6 @@ import java.sql.Connection; import java.sql.SQLException; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.function.Supplier; import javax.sql.DataSource; @@ -70,14 +68,14 @@ public class JdbcMetaStoreManagerFactory implements MetaStoreManagerFactory { final Map storageCredentialCacheMap = new HashMap<>(); final Map entityCacheMap = new HashMap<>(); final Map> sessionSupplierMap = new HashMap<>(); - final List dataSources; + final DataSource dataSource; protected final PolarisDiagnostics diagServices = new PolarisDefaultDiagServiceImpl(); @Inject PolarisStorageIntegrationProvider storageIntegrationProvider; @Inject - protected JdbcMetaStoreManagerFactory(List dataSources) { - this.dataSources = dataSources; + protected JdbcMetaStoreManagerFactory(DataSource dataSource) { + this.dataSource = dataSource; } protected PrincipalSecretsGenerator secretsGenerator( @@ -112,12 +110,11 @@ private void initializeForRealm( private DatasourceOperations getDatasourceOperations(boolean isBootstrap) { // TODO: remove when multiple dataSources are supported. - Preconditions.checkState(dataSources.size() == 1, "More than one dataSources configured"); - DatasourceOperations databaseOperations = new DatasourceOperations(dataSources.getFirst()); + DatasourceOperations databaseOperations = new DatasourceOperations(dataSource); if (isBootstrap) { try { DatabaseType databaseType; - try (Connection connection = dataSources.getFirst().getConnection()) { + try (Connection connection = dataSource.getConnection()) { String productName = connection.getMetaData().getDatabaseProductName(); databaseType = DatabaseType.fromDisplayName(productName); } diff --git a/quarkus/admin/src/main/java/org/apache/polaris/admintool/config/QuarkusProducers.java b/quarkus/admin/src/main/java/org/apache/polaris/admintool/config/QuarkusProducers.java index 69161419ee..07ad023629 100644 --- a/quarkus/admin/src/main/java/org/apache/polaris/admintool/config/QuarkusProducers.java +++ b/quarkus/admin/src/main/java/org/apache/polaris/admintool/config/QuarkusProducers.java @@ -18,8 +18,6 @@ */ package org.apache.polaris.admintool.config; -import io.quarkus.arc.All; -import io.quarkus.arc.InstanceHandle; import io.smallrye.common.annotation.Identifier; import jakarta.annotation.Nullable; import jakarta.enterprise.context.ApplicationScoped; @@ -27,9 +25,6 @@ import jakarta.enterprise.inject.Instance; import jakarta.enterprise.inject.Produces; import java.time.Clock; -import java.util.ArrayList; -import java.util.List; -import javax.sql.DataSource; import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.config.PolarisConfigurationStore; @@ -81,13 +76,4 @@ public PolarisConfigurationStore configurationStore() { // A configuration store is not required when running the admin tool. return new PolarisConfigurationStore() {}; } - - @Produces - public List dataSources(@All List> dataSources) { - List dataSourceList = new ArrayList<>(); - for (InstanceHandle handle : dataSources) { - dataSourceList.add(handle.get()); - } - return dataSourceList; - } } diff --git a/quarkus/server/build.gradle.kts b/quarkus/server/build.gradle.kts index 61c1b46660..cf55d53751 100644 --- a/quarkus/server/build.gradle.kts +++ b/quarkus/server/build.gradle.kts @@ -50,6 +50,12 @@ dependencies { runtimeOnly(project(":polaris-eclipselink")) runtimeOnly("org.postgresql:postgresql") + runtimeOnly(project(":polaris-relational-jdbc")) + runtimeOnly("io.quarkus:quarkus-jdbc-postgresql") { + exclude(group = "org.antlr", module = "antlr4-runtime") + exclude(group = "org.scala-lang", module = "scala-library") + exclude(group = "org.scala-lang", module = "scala-reflect") + } // enforce the Quarkus _platform_ here, to get a consistent and validated set of dependencies implementation(enforcedPlatform(libs.quarkus.bom)) diff --git a/quarkus/service/build.gradle.kts b/quarkus/service/build.gradle.kts index d2375b8ef6..97b3daa0de 100644 --- a/quarkus/service/build.gradle.kts +++ b/quarkus/service/build.gradle.kts @@ -125,11 +125,6 @@ dependencies { testImplementation(libs.hawkular.agent.prometheus.scraper) testImplementation(project(":polaris-quarkus-test-commons")) - testFixturesApi(enforcedPlatform(libs.quarkus.bom)) { - exclude(group = "org.antlr", module = "antlr4-runtime") - exclude(group = "org.scala-lang", module = "scala-library") - exclude(group = "org.scala-lang", module = "scala-reflect") - } testImplementation("io.quarkus:quarkus-junit5") testImplementation(platform(libs.testcontainers.bom)) testImplementation("org.testcontainers:testcontainers") diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java index cbede0a1e9..625e57e45d 100644 --- a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java @@ -18,8 +18,6 @@ */ package org.apache.polaris.service.quarkus.config; -import io.quarkus.arc.All; -import io.quarkus.arc.InstanceHandle; import io.quarkus.runtime.StartupEvent; import io.smallrye.common.annotation.Identifier; import io.smallrye.context.SmallRyeManagedExecutor; @@ -34,9 +32,6 @@ import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.core.Context; import java.time.Clock; -import java.util.ArrayList; -import java.util.List; -import javax.sql.DataSource; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.PolarisDiagnostics; @@ -266,13 +261,4 @@ public ActiveRolesProvider activeRolesProvider( public void closeTaskExecutor(@Disposes @Identifier("task-executor") ManagedExecutor executor) { executor.close(); } - - @Produces - public List dataSources(@All List> dataSources) { - List dataSourceList = new ArrayList<>(); - for (InstanceHandle handle : dataSources) { - dataSourceList.add(handle.get()); - } - return dataSourceList; - } } From 5c9f323ed17b4b6b3f82dfb092aa7854d3b774de Mon Sep 17 00:00:00 2001 From: Prashant Date: Fri, 25 Apr 2025 17:43:28 -0700 Subject: [PATCH 09/11] Revert single DS from constructor --- .../jdbc/JdbcMetaStoreManagerFactory.java | 13 ++++++++----- .../polaris/admintool/config/QuarkusProducers.java | 14 ++++++++++++++ .../service/quarkus/config/QuarkusProducers.java | 14 ++++++++++++++ 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java b/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java index 23860207d2..2aaafd5728 100644 --- a/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java +++ b/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java @@ -18,6 +18,7 @@ */ package org.apache.polaris.extension.persistence.relational.jdbc; +import com.google.common.base.Preconditions; import io.smallrye.common.annotation.Identifier; import jakarta.annotation.Nullable; import jakarta.enterprise.context.ApplicationScoped; @@ -25,6 +26,7 @@ import java.sql.Connection; import java.sql.SQLException; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.function.Supplier; import javax.sql.DataSource; @@ -68,14 +70,14 @@ public class JdbcMetaStoreManagerFactory implements MetaStoreManagerFactory { final Map storageCredentialCacheMap = new HashMap<>(); final Map entityCacheMap = new HashMap<>(); final Map> sessionSupplierMap = new HashMap<>(); - final DataSource dataSource; + final List dataSources; protected final PolarisDiagnostics diagServices = new PolarisDefaultDiagServiceImpl(); @Inject PolarisStorageIntegrationProvider storageIntegrationProvider; @Inject - protected JdbcMetaStoreManagerFactory(DataSource dataSource) { - this.dataSource = dataSource; + protected JdbcMetaStoreManagerFactory(List dataSources) { + this.dataSources = dataSources; } protected PrincipalSecretsGenerator secretsGenerator( @@ -110,11 +112,12 @@ private void initializeForRealm( private DatasourceOperations getDatasourceOperations(boolean isBootstrap) { // TODO: remove when multiple dataSources are supported. - DatasourceOperations databaseOperations = new DatasourceOperations(dataSource); + Preconditions.checkState(dataSources.size() == 1, "More than one dataSources configured"); + DatasourceOperations databaseOperations = new DatasourceOperations(dataSources.getFirst()); if (isBootstrap) { try { DatabaseType databaseType; - try (Connection connection = dataSource.getConnection()) { + try (Connection connection = dataSources.getFirst().getConnection()) { String productName = connection.getMetaData().getDatabaseProductName(); databaseType = DatabaseType.fromDisplayName(productName); } diff --git a/quarkus/admin/src/main/java/org/apache/polaris/admintool/config/QuarkusProducers.java b/quarkus/admin/src/main/java/org/apache/polaris/admintool/config/QuarkusProducers.java index 07ad023629..69161419ee 100644 --- a/quarkus/admin/src/main/java/org/apache/polaris/admintool/config/QuarkusProducers.java +++ b/quarkus/admin/src/main/java/org/apache/polaris/admintool/config/QuarkusProducers.java @@ -18,6 +18,8 @@ */ package org.apache.polaris.admintool.config; +import io.quarkus.arc.All; +import io.quarkus.arc.InstanceHandle; import io.smallrye.common.annotation.Identifier; import jakarta.annotation.Nullable; import jakarta.enterprise.context.ApplicationScoped; @@ -25,6 +27,9 @@ import jakarta.enterprise.inject.Instance; import jakarta.enterprise.inject.Produces; import java.time.Clock; +import java.util.ArrayList; +import java.util.List; +import javax.sql.DataSource; import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.config.PolarisConfigurationStore; @@ -76,4 +81,13 @@ public PolarisConfigurationStore configurationStore() { // A configuration store is not required when running the admin tool. return new PolarisConfigurationStore() {}; } + + @Produces + public List dataSources(@All List> dataSources) { + List dataSourceList = new ArrayList<>(); + for (InstanceHandle handle : dataSources) { + dataSourceList.add(handle.get()); + } + return dataSourceList; + } } diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java index 625e57e45d..cbede0a1e9 100644 --- a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java @@ -18,6 +18,8 @@ */ package org.apache.polaris.service.quarkus.config; +import io.quarkus.arc.All; +import io.quarkus.arc.InstanceHandle; import io.quarkus.runtime.StartupEvent; import io.smallrye.common.annotation.Identifier; import io.smallrye.context.SmallRyeManagedExecutor; @@ -32,6 +34,9 @@ import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.core.Context; import java.time.Clock; +import java.util.ArrayList; +import java.util.List; +import javax.sql.DataSource; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.PolarisDiagnostics; @@ -261,4 +266,13 @@ public ActiveRolesProvider activeRolesProvider( public void closeTaskExecutor(@Disposes @Identifier("task-executor") ManagedExecutor executor) { executor.close(); } + + @Produces + public List dataSources(@All List> dataSources) { + List dataSourceList = new ArrayList<>(); + for (InstanceHandle handle : dataSources) { + dataSourceList.add(handle.get()); + } + return dataSourceList; + } } From 9bf2972e97df71ea2cf05af419e8bbdf6066216a Mon Sep 17 00:00:00 2001 From: Prashant Date: Fri, 25 Apr 2025 18:24:28 -0700 Subject: [PATCH 10/11] remove the exlusion --- quarkus/server/build.gradle.kts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/quarkus/server/build.gradle.kts b/quarkus/server/build.gradle.kts index cf55d53751..70112fd06d 100644 --- a/quarkus/server/build.gradle.kts +++ b/quarkus/server/build.gradle.kts @@ -51,11 +51,7 @@ dependencies { runtimeOnly(project(":polaris-eclipselink")) runtimeOnly("org.postgresql:postgresql") runtimeOnly(project(":polaris-relational-jdbc")) - runtimeOnly("io.quarkus:quarkus-jdbc-postgresql") { - exclude(group = "org.antlr", module = "antlr4-runtime") - exclude(group = "org.scala-lang", module = "scala-library") - exclude(group = "org.scala-lang", module = "scala-reflect") - } + runtimeOnly("io.quarkus:quarkus-jdbc-postgresql") // enforce the Quarkus _platform_ here, to get a consistent and validated set of dependencies implementation(enforcedPlatform(libs.quarkus.bom)) From 6835385244a0747241ee19bae32b8fcd8f609eff Mon Sep 17 00:00:00 2001 From: Prashant Date: Fri, 25 Apr 2025 18:46:59 -0700 Subject: [PATCH 11/11] Address review feedback --- .../jdbc/JdbcBasePersistenceImpl.java | 27 +++++++++---------- .../jdbc/JdbcMetaStoreManagerFactory.java | 18 +++++-------- .../admintool/config/QuarkusProducers.java | 14 ---------- .../quarkus/config/QuarkusProducers.java | 14 ---------- 4 files changed, 19 insertions(+), 54 deletions(-) diff --git a/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcBasePersistenceImpl.java b/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcBasePersistenceImpl.java index 877ac62dd4..ea99ed98da 100644 --- a/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcBasePersistenceImpl.java +++ b/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcBasePersistenceImpl.java @@ -87,7 +87,15 @@ public void writeEntity( @Nonnull PolarisBaseEntity entity, boolean nameOrParentChanged, PolarisBaseEntity originalEntity) { - persistEntity(callCtx, entity, originalEntity, datasourceOperations); + try { + datasourceOperations.runWithinTransaction( + statement -> { + persistEntity(callCtx, entity, originalEntity, statement); + return true; + }); + } catch (SQLException e) { + throw new RuntimeException("Error persisting entity", e); + } } @Override @@ -130,11 +138,12 @@ private void persistEntity( @Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity, PolarisBaseEntity originalEntity, - Object executor) { + Statement statement) + throws SQLException { ModelEntity modelEntity = ModelEntity.fromEntity(entity); if (originalEntity == null) { try { - execute(executor, generateInsertQuery(modelEntity, realmId)); + statement.executeUpdate(generateInsertQuery(modelEntity, realmId)); } catch (SQLException e) { if (datasourceOperations.isConstraintViolation(e)) { PolarisBaseEntity existingEntity = @@ -162,7 +171,7 @@ private void persistEntity( "realm_id", realmId); try { - int rowsUpdated = execute(executor, generateUpdateQuery(modelEntity, params)); + int rowsUpdated = statement.executeUpdate(generateUpdateQuery(modelEntity, params)); if (rowsUpdated == 0) { throw new RetryOnConcurrencyException( "Entity '%s' id '%s' concurrently modified; expected version %s", @@ -175,16 +184,6 @@ private void persistEntity( } } - private int execute(Object executor, String query) throws SQLException { - if (executor instanceof Statement) { - return ((Statement) executor).executeUpdate(query); - } else if (executor instanceof DatasourceOperations) { - return ((DatasourceOperations) executor).executeUpdate(query); - } else { - throw new IllegalArgumentException("Unsupported executor: " + executor); - } - } - @Override public void writeToGrantRecords( @Nonnull PolarisCallContext callCtx, @Nonnull PolarisGrantRecord grantRec) { diff --git a/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java b/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java index 2aaafd5728..0f981a99b2 100644 --- a/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java +++ b/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java @@ -18,15 +18,14 @@ */ package org.apache.polaris.extension.persistence.relational.jdbc; -import com.google.common.base.Preconditions; import io.smallrye.common.annotation.Identifier; import jakarta.annotation.Nullable; import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.inject.Instance; import jakarta.inject.Inject; import java.sql.Connection; import java.sql.SQLException; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.function.Supplier; import javax.sql.DataSource; @@ -70,15 +69,12 @@ public class JdbcMetaStoreManagerFactory implements MetaStoreManagerFactory { final Map storageCredentialCacheMap = new HashMap<>(); final Map entityCacheMap = new HashMap<>(); final Map> sessionSupplierMap = new HashMap<>(); - final List dataSources; - protected final PolarisDiagnostics diagServices = new PolarisDefaultDiagServiceImpl(); + @Inject PolarisStorageIntegrationProvider storageIntegrationProvider; + @Inject Instance dataSource; - @Inject - protected JdbcMetaStoreManagerFactory(List dataSources) { - this.dataSources = dataSources; - } + protected JdbcMetaStoreManagerFactory() {} protected PrincipalSecretsGenerator secretsGenerator( RealmContext realmContext, @Nullable RootCredentialsSet rootCredentialsSet) { @@ -111,13 +107,11 @@ private void initializeForRealm( } private DatasourceOperations getDatasourceOperations(boolean isBootstrap) { - // TODO: remove when multiple dataSources are supported. - Preconditions.checkState(dataSources.size() == 1, "More than one dataSources configured"); - DatasourceOperations databaseOperations = new DatasourceOperations(dataSources.getFirst()); + DatasourceOperations databaseOperations = new DatasourceOperations(dataSource.get()); if (isBootstrap) { try { DatabaseType databaseType; - try (Connection connection = dataSources.getFirst().getConnection()) { + try (Connection connection = dataSource.get().getConnection()) { String productName = connection.getMetaData().getDatabaseProductName(); databaseType = DatabaseType.fromDisplayName(productName); } diff --git a/quarkus/admin/src/main/java/org/apache/polaris/admintool/config/QuarkusProducers.java b/quarkus/admin/src/main/java/org/apache/polaris/admintool/config/QuarkusProducers.java index 69161419ee..07ad023629 100644 --- a/quarkus/admin/src/main/java/org/apache/polaris/admintool/config/QuarkusProducers.java +++ b/quarkus/admin/src/main/java/org/apache/polaris/admintool/config/QuarkusProducers.java @@ -18,8 +18,6 @@ */ package org.apache.polaris.admintool.config; -import io.quarkus.arc.All; -import io.quarkus.arc.InstanceHandle; import io.smallrye.common.annotation.Identifier; import jakarta.annotation.Nullable; import jakarta.enterprise.context.ApplicationScoped; @@ -27,9 +25,6 @@ import jakarta.enterprise.inject.Instance; import jakarta.enterprise.inject.Produces; import java.time.Clock; -import java.util.ArrayList; -import java.util.List; -import javax.sql.DataSource; import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.config.PolarisConfigurationStore; @@ -81,13 +76,4 @@ public PolarisConfigurationStore configurationStore() { // A configuration store is not required when running the admin tool. return new PolarisConfigurationStore() {}; } - - @Produces - public List dataSources(@All List> dataSources) { - List dataSourceList = new ArrayList<>(); - for (InstanceHandle handle : dataSources) { - dataSourceList.add(handle.get()); - } - return dataSourceList; - } } diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java index cbede0a1e9..625e57e45d 100644 --- a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java @@ -18,8 +18,6 @@ */ package org.apache.polaris.service.quarkus.config; -import io.quarkus.arc.All; -import io.quarkus.arc.InstanceHandle; import io.quarkus.runtime.StartupEvent; import io.smallrye.common.annotation.Identifier; import io.smallrye.context.SmallRyeManagedExecutor; @@ -34,9 +32,6 @@ import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.core.Context; import java.time.Clock; -import java.util.ArrayList; -import java.util.List; -import javax.sql.DataSource; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.PolarisDiagnostics; @@ -266,13 +261,4 @@ public ActiveRolesProvider activeRolesProvider( public void closeTaskExecutor(@Disposes @Identifier("task-executor") ManagedExecutor executor) { executor.close(); } - - @Produces - public List dataSources(@All List> dataSources) { - List dataSourceList = new ArrayList<>(); - for (InstanceHandle handle : dataSources) { - dataSourceList.add(handle.get()); - } - return dataSourceList; - } }