NoSQL persistence

Author: snazy

LICENSE

@@ -230,6 +230,16 @@ License: https://www.apache.org/licenses/LICENSE-2.0
 
 --------------------------------------------------------------------------------
 
+This product includes code from Netty.
+
+* persistence/nosql/persistence/cdi/quarkus-distcache/src/main/java/org/apache/polaris/persistence/nosql/quarkus/distcache/ResolvConf.java
+
+Copyright: Copyright © 2025 The Netty project
+Home page: https://netty.io/
+License: https://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
 This product includes code from OpenAPITool openapi-generator
 
 * server-templates/formParams.mustache
@@ -328,6 +338,9 @@ This product includes code from Project Nessie.
 * helm/polaris/templates/servicemonitor.yaml
 * helm/polaris/templates/storage.yaml
 
+Code underneath the components/persistence directory, especially pluggable object types, index related, cache,
+atomic commit logic and fundamental persistence implementations.
+
 Copyright: Copyright 2015-2025 Dremio Corporation
 Home page: https://projectnessie.org/
 License: https://www.apache.org/licenses/LICENSE-2.0

bom/build.gradle.kts

@@ -47,6 +47,38 @@ dependencies {
     api(project(":polaris-nodes-api"))
     api(project(":polaris-nodes-impl"))
     api(project(":polaris-nodes-spi"))
+    api(project(":polaris-nodes-store-nosql"))
+
+    api(project(":polaris-realms-api"))
+    api(project(":polaris-realms-impl"))
+    api(project(":polaris-realms-spi"))
+    api(project(":polaris-realms-store-nosql"))
+
+    api(project(":polaris-authz-api"))
+    api(project(":polaris-authz-impl"))
+    api(project(":polaris-authz-spi"))
+    api(project(":polaris-authz-store-nosql"))
+
+    api(project(":polaris-persistence-nosql-api"))
+    api(project(":polaris-persistence-nosql-impl"))
+    api(project(":polaris-persistence-nosql-benchmark"))
+    api(project(":polaris-persistence-nosql-metastore"))
+    api(project(":polaris-persistence-nosql-cdi-common"))
+    api(project(":polaris-persistence-nosql-cdi-quarkus"))
+    api(project(":polaris-persistence-nosql-cdi-quarkus-distcache"))
+    api(project(":polaris-persistence-nosql-cdi-weld"))
+    api(project(":polaris-persistence-nosql-correctness"))
+    api(project(":polaris-persistence-nosql-standalone"))
+    api(project(":polaris-persistence-nosql-testextension"))
+    api(project(":polaris-persistence-nosql-types"))
+
+    api(project(":polaris-persistence-nosql-inmemory"))
+    api(project(":polaris-persistence-nosql-mongodb"))
+
+    api(project(":polaris-persistence-nosql-maintenance-api"))
+    api(project(":polaris-persistence-nosql-maintenance-impl"))
+    api(project(":polaris-persistence-nosql-maintenance-cel"))
+    api(project(":polaris-persistence-nosql-maintenance-spi"))
 
     api(project(":polaris-persistence-nosql-api"))
 

build.gradle.kts

@@ -92,6 +92,11 @@ tasks.named<RatTask>("rat").configure {
   excludes.add("logs/**")
   excludes.add("**/*.lock")
 
+  // Binary files
+  excludes.add(
+    "persistence/nosql/persistence/index/src/testFixtures/resources/org/apache/polaris/persistence/indexes/words.gz"
+  )
+
   // Polaris service startup banner
   excludes.add("runtime/service/src/**/banner.txt")
 

codestyle/checkstyle.xml

@@ -43,7 +43,7 @@
     <!-- Checks for imports                            -->
     <!-- See http://checkstyle.org/config_imports.html -->
     <module name="IllegalImport">
-      <property name="illegalPkgs" value=".*\.shaded\..*, .*\.relocated\..*"/>
+      <property name="illegalPkgs" value=".*\.shaded\..*, (?!org\.projectnessie\.cel).*\.relocated\..*"/>
       <property name="regexp" value="true"/>
     </module>
 

gradle/libs.versions.toml

@@ -39,7 +39,7 @@ swagger = "1.6.16"
 # If a dependency is removed, check whether the LICENSE and/or NOTICE files need to be adopted
 # (aka mention of the dependency removed).
 #
-quarkus-amazon-services-bom = { module = "io.quarkus.platform:quarkus-amazon-services-bom", version.ref="quarkus" }
+agrona = { module = "org.agrona:agrona", version = "2.2.4" }
 antlr4-runtime = { module = "org.antlr:antlr4-runtime", version.strictly = "4.9.3" } # spark integration tests
 apache-httpclient5 = { module = "org.apache.httpcomponents.client5:httpclient5", version = "5.5.1" }
 assertj-core = { module = "org.assertj:assertj-core", version = "3.27.6" }
@@ -48,8 +48,10 @@ awssdk-bom = { module = "software.amazon.awssdk:bom", version = "2.37.3" }
 awaitility = { module = "org.awaitility:awaitility", version = "4.3.0" }
 azuresdk-bom = { module = "com.azure:azure-sdk-bom", version = "1.3.0" }
 caffeine = { module = "com.github.ben-manes.caffeine:caffeine", version = "3.2.3" }
+cel-bom = { module = "org.projectnessie.cel:cel-bom", version = "0.5.3" }
 commons-lang3 = { module = "org.apache.commons:commons-lang3", version = "3.19.0" }
 commons-text = { module = "org.apache.commons:commons-text", version = "1.14.0" }
+docker-java-api = { module = "com.github.docker-java:docker-java-api", version = "3.6.0" }
 errorprone = { module = "com.google.errorprone:error_prone_core", version = "2.43.0" }
 google-cloud-storage-bom = { module = "com.google.cloud:google-cloud-storage-bom", version = "2.59.0" }
 guava = { module = "com.google.guava:guava", version = "33.5.0-jre" }
@@ -74,23 +76,26 @@ jakarta-servlet-api = { module = "jakarta.servlet:jakarta.servlet-api", version
 jakarta-validation-api = { module = "jakarta.validation:jakarta.validation-api", version = "3.1.1" }
 jakarta-ws-rs-api = { module = "jakarta.ws.rs:jakarta.ws.rs-api", version = "4.0.0" }
 javax-servlet-api = { module = "javax.servlet:javax.servlet-api", version = "4.0.1" }
-junit-bom = { module = "org.junit:junit-bom", version = "5.14.1" }
-junit-pioneer = { module = "org.junit-pioneer:junit-pioneer", version = "2.3.0" }
-keycloak-admin-client = { module = "org.keycloak:keycloak-admin-client", version = "26.0.7" }
 jcstress-core = { module = "org.openjdk.jcstress:jcstress-core", version = "0.16" }
 jmh-core = { module = "org.openjdk.jmh:jmh-core", version.ref = "jmh" }
 jmh-generator-annprocess = { module = "org.openjdk.jmh:jmh-generator-annprocess", version.ref = "jmh" }
+junit-bom = { module = "org.junit:junit-bom", version = "5.14.1" }
+junit-pioneer = { module = "org.junit-pioneer:junit-pioneer", version = "2.3.0" }
+keycloak-admin-client = { module = "org.keycloak:keycloak-admin-client", version = "26.0.7" }
 logback-classic = { module = "ch.qos.logback:logback-classic", version = "1.5.20" }
 micrometer-bom = { module = "io.micrometer:micrometer-bom", version = "1.15.5" }
 microprofile-fault-tolerance-api = { module = "org.eclipse.microprofile.fault-tolerance:microprofile-fault-tolerance-api", version = "4.1.2" }
 mockito-core = { module = "org.mockito:mockito-core", version = "5.20.0" }
 mockito-junit-jupiter = { module = "org.mockito:mockito-junit-jupiter", version = "5.20.0" }
+mongodb-driver-sync = { module = "org.mongodb:mongodb-driver-sync", version = "5.6.1" }
 opentelemetry-bom = { module = "io.opentelemetry:opentelemetry-bom", version = "1.55.0" }
+opentelemetry-instrumentation-bom-alpha = { module = "io.opentelemetry.instrumentation:opentelemetry-instrumentation-bom-alpha", version= "2.20.1-alpha" }
 opentelemetry-semconv = { module = "io.opentelemetry.semconv:opentelemetry-semconv", version = "1.37.0" }
 picocli = { module = "info.picocli:picocli-codegen", version.ref = "picocli" }
 picocli-codegen = { module = "info.picocli:picocli-codegen", version.ref = "picocli" }
 postgresql = { module = "org.postgresql:postgresql", version = "42.7.8" }
 prometheus-metrics-exporter-servlet-jakarta = { module = "io.prometheus:prometheus-metrics-exporter-servlet-jakarta", version = "1.4.2" }
+quarkus-amazon-services-bom = { module = "io.quarkus.platform:quarkus-amazon-services-bom", version.ref="quarkus" }
 quarkus-bom = { module = "io.quarkus.platform:quarkus-bom", version.ref = "quarkus" }
 scala212-lang-library = { module = "org.scala-lang:scala-library", version.ref = "scala212" }
 scala212-lang-reflect = { module = "org.scala-lang:scala-reflect", version.ref = "scala212" }

gradle/projects.main.properties

@@ -62,6 +62,37 @@ polaris-idgen-spi=persistence/nosql/idgen/spi
 polaris-nodes-api=persistence/nosql/nodes/api
 polaris-nodes-impl=persistence/nosql/nodes/impl
 polaris-nodes-spi=persistence/nosql/nodes/spi
+polaris-nodes-store-nosql=persistence/nosql/nodes/store-nosql
+# realms
+polaris-realms-api=persistence/nosql/realms/api
+polaris-realms-impl=persistence/nosql/realms/impl
+polaris-realms-spi=persistence/nosql/realms/spi
+polaris-realms-store-nosql=persistence/nosql/realms/store-nosql
+# authz
+polaris-authz-api=persistence/nosql/authz/api
+polaris-authz-impl=persistence/nosql/authz/impl
+polaris-authz-spi=persistence/nosql/authz/spi
+polaris-authz-store-nosql=persistence/nosql/authz/store-nosql
 # persistence / database agnostic
 polaris-persistence-nosql-api=persistence/nosql/persistence/api
+polaris-persistence-nosql-impl=persistence/nosql/persistence/impl
+polaris-persistence-nosql-benchmark=persistence/nosql/persistence/benchmark
+polaris-persistence-nosql-metastore=persistence/nosql/persistence/metastore
+polaris-persistence-nosql-cdi-common=persistence/nosql/persistence/cdi/common
+polaris-persistence-nosql-cdi-quarkus=persistence/nosql/persistence/cdi/quarkus
+polaris-persistence-nosql-cdi-quarkus-distcache=persistence/nosql/persistence/cdi/quarkus-distcache
+polaris-persistence-nosql-cdi-weld=persistence/nosql/persistence/cdi/weld
+polaris-persistence-nosql-correctness=persistence/nosql/persistence/correctness
+polaris-persistence-nosql-standalone=persistence/nosql/persistence/standalone
+polaris-persistence-nosql-testextension=persistence/nosql/persistence/testextension
+polaris-persistence-nosql-types=persistence/nosql/persistence/types
 polaris-persistence-nosql-varint=persistence/nosql/persistence/varint
+# persistence / maintenance
+polaris-persistence-nosql-maintenance-api=persistence/nosql/persistence/maintenance/api
+polaris-persistence-nosql-maintenance-impl=persistence/nosql/persistence/maintenance/impl
+polaris-persistence-nosql-maintenance-cel=persistence/nosql/persistence/maintenance/retain-cel
+polaris-persistence-nosql-maintenance-spi=persistence/nosql/persistence/maintenance/spi
+# persistence / database specific implementations
+polaris-persistence-nosql-inmemory=persistence/nosql/persistence/db/inmemory
+polaris-persistence-nosql-mongodb=persistence/nosql/persistence/db/mongodb
+

persistence/nosql/authz/README.md

@@ -0,0 +1,64 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+ 
+   http://www.apache.org/licenses/LICENSE-2.0
+ 
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# AuthZ framework with pluggable privileges
+
+Provides a framework and implementations pluggable privileges and privilege checks.
+
+## Privileges
+
+A privilege is globally identified by its name. Privileges can be inheritable (from its parents) or not. Multiple
+privileges can be grouped together to a _composite_ privilege (think: `ALL_DML` having `SELECT`, `INSERT`, `UPDATE` and
+`DELETE`) - a composite privilege matches, if all its individual privileges match. Multiple privileges can also be
+grouped to an _alternative_ privilege, which matches if any of its individual privileges matches.
+
+Available privileges are provided by one or more `PrivilegeProvider`s, which are discovered at runtime.
+Note: currently there is only one `ProvilegeProvider` that plugs in the Polaris privileges.
+
+## ACLs, ACL entries and ACL chains
+
+Each securable object can have its own ACL. ACLs consist of ACL entries, which define the _granted_ and _restricted_
+privileges by role name. The the number of roles is technically unbounded and the number of ACL entries can become
+quite large.
+
+This framework implements [separation of duties](https://en.wikipedia.org/wiki/Separation_of_duties) ("SoD"), which is a
+quite demanded functionality not just by large(r) user organizations. TL;DR _SoD_ allows "security administrators" to
+grant and revoke privileges to other users, but not leverage those privileges themselves.
+
+The _effective_ set of privileges for a specific operation performed by a specific caller needs to be computed against
+the target objects and their parents. _ACL chains_ are the vehicle to model this hierarchy and let the implementation
+compute the set of _effective_ privileges based on the individual ACLs and roles.
+
+Note: Privilege checks and _SoD_ are currently not performed via this framework.
+
+## Jackson support & Storage friendly representation
+
+The persistable types `Acl`, `AclEntry`, and `PrivilegeSet` can all be serialized using Jackson.
+
+As the number of ACL entries can become quite large, space efficient serialization is quite important. The
+implementation uses bit-set encoding when serializing `PrivilegeSet`s for persistence.
+
+## Code structure
+
+The code is structured into multiple modules. Consuming code should almost always pull in only the API module.
+
+* `polaris-authz-api` provides the necessary Java interfaces and immutable types.
+* `polaris-authz-impl` provides the storage agnostic implementation.
+* `polaris-authz-spi` provides the necessary interfaces to provide custom privileges and storage implementation.
+* `polaris-authz-store-nosql` provides the storage implementation based on `polaris-persistence-nosql-api`.

persistence/nosql/authz/api/build.gradle.kts

@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+plugins {
+  id("org.kordamp.gradle.jandex")
+  id("polaris-server")
+}
+
+description = "Polaris AuthZ API"
+
+dependencies {
+  implementation(libs.guava)
+
+  implementation(platform(libs.jackson.bom))
+  implementation("com.fasterxml.jackson.core:jackson-databind")
+
+  compileOnly(libs.jakarta.annotation.api)
+  compileOnly(libs.jakarta.validation.api)
+  compileOnly(libs.jakarta.inject.api)
+  compileOnly(libs.jakarta.enterprise.cdi.api)
+
+  compileOnly(project(":polaris-immutables"))
+  annotationProcessor(project(":polaris-immutables", configuration = "processor"))
+}

persistence/nosql/authz/api/src/main/java/org/apache/polaris/authz/api/Acl.java

@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.authz.api;
+
+import com.google.errorprone.annotations.CanIgnoreReturnValue;
+import jakarta.annotation.Nonnull;
+import java.util.Set;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+
+public interface Acl {
+
+  void entriesForRoleIds(
+      @Nonnull Set<String> roleIds, @Nonnull Consumer<AclEntry> aclEntryConsumer);
+
+  void forEach(@Nonnull BiConsumer<String, AclEntry> consumer);
+
+  interface AclBuilder {
+    @CanIgnoreReturnValue
+    AclBuilder from(@Nonnull Acl instance);
+
+    @CanIgnoreReturnValue
+    AclBuilder addEntry(@Nonnull String roleId, @Nonnull AclEntry entry);
+
+    @CanIgnoreReturnValue
+    AclBuilder removeEntry(@Nonnull String roleId);
+
+    /**
+     * Add, remove or update an {@linkplain AclEntry ACL entry} for a role.
+     *
+     * <p>The {@linkplain Consumer consumer} is called with an empty builder, if no ACL entry for
+     * the role exists, otherwise with a builder constructed from the existing entry. If the given
+     * {@linkplain Consumer consumer} removes all privileges from the ACL entry, the ACL entry will
+     * be removed.
+     */
+    @CanIgnoreReturnValue
+    AclBuilder modify(@Nonnull String roleId, @Nonnull Consumer<AclEntry.AclEntryBuilder> entry);
+
+    Acl build();
+  }
+}

persistence/nosql/authz/api/src/main/java/org/apache/polaris/authz/api/AclChain.java

@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.authz.api;
+
+import java.util.Optional;
+import org.apache.polaris.immutables.PolarisImmutable;
+import org.immutables.value.Value;
+
+/** Container for an {@linkplain Acl ACL} of an individual entity and a pointer to its parent. */
+@PolarisImmutable
+public interface AclChain {
+  @Value.Parameter(order = 1)
+  Acl acl();
+
+  @Value.Parameter(order = 2)
+  Optional<AclChain> parent();
+
+  static AclChain aclChain(Acl acl, Optional<AclChain> parent) {
+    return ImmutableAclChain.of(acl, parent);
+  }
+}