NoSQL: Node IDs - API, SPI + general implementation

This PR provides a mechanism to assign a Polaris-cluster-wide unique node-ID to each Polaris instance, which is then used when generating Polaris-cluster-wide unique Snowflake-IDs.

The change is fundamental for the NoSQL work, but also demanded for the existing relational JDBC persistence.

Does not include any persistence specific implementation.

Author: snazy

bom/build.gradle.kts

@@ -44,6 +44,10 @@ dependencies {
     api(project(":polaris-idgen-impl"))
     api(project(":polaris-idgen-spi"))
 
+    api(project(":polaris-nodes-api"))
+    api(project(":polaris-nodes-impl"))
+    api(project(":polaris-nodes-spi"))
+
     api(project(":polaris-config-docs-annotations"))
     api(project(":polaris-config-docs-generator"))
 

gradle/projects.main.properties

@@ -57,5 +57,9 @@ polaris-idgen-api=persistence/nosql/idgen/api
 polaris-idgen-impl=persistence/nosql/idgen/impl
 polaris-idgen-mocks=persistence/nosql/idgen/mocks
 polaris-idgen-spi=persistence/nosql/idgen/spi
+# nodes
+polaris-nodes-api=persistence/nosql/nodes/api
+polaris-nodes-impl=persistence/nosql/nodes/impl
+polaris-nodes-spi=persistence/nosql/nodes/spi
 # persistence / database agnostic
 polaris-persistence-nosql-varint=persistence/nosql/persistence/varint

persistence/nosql/nodes/README.md

@@ -0,0 +1,34 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+ 
+   http://www.apache.org/licenses/LICENSE-2.0
+ 
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# Uniquely identify running Polaris nodes
+
+Some ID generation mechanisms,
+like [Snowflake-IDs](https://medium.com/@jitenderkmr/demystifying-snowflake-ids-a-unique-identifier-in-distributed-computing-72796a827c9d),
+require unique integer IDs for each running node. This framework provides a mechanism to assign each running node a
+unique integer ID.
+
+## Code structure
+
+The code is structured into multiple modules. Consuming code should almost always pull in only the API module.
+
+* `polaris-nodes-api` provides the necessary Java interfaces and immutable types.
+* `polaris-nodes-impl` provides the storage agnostic implementation.
+* `polaris-nodes-spi` provides the necessary interfaces to provide a storage specific implementation.
+* `polaris-nodes-store-nosql` provides the storage implementation based on `polaris-persistence-nosql-api`.

persistence/nosql/nodes/api/build.gradle.kts

@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+plugins {
+  id("org.kordamp.gradle.jandex")
+  id("polaris-server")
+}
+
+description = "Polaris nodes API, no concrete implementations"
+
+dependencies {
+  implementation(project(":polaris-idgen-api"))
+
+  compileOnly(project(":polaris-immutables"))
+  annotationProcessor(project(":polaris-immutables", configuration = "processor"))
+
+  implementation(platform(libs.jackson.bom))
+  implementation("com.fasterxml.jackson.core:jackson-databind")
+
+  compileOnly(libs.jakarta.annotation.api)
+  compileOnly(libs.jakarta.validation.api)
+  compileOnly(libs.jakarta.inject.api)
+  compileOnly(libs.jakarta.enterprise.cdi.api)
+
+  compileOnly(libs.smallrye.config.core)
+  compileOnly(platform(libs.quarkus.bom))
+  compileOnly("io.quarkus:quarkus-core")
+}

persistence/nosql/nodes/api/src/main/java/org/apache/polaris/nodes/api/Node.java

@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.nodes.api;
+
+import jakarta.annotation.Nullable;
+import java.time.Instant;
+import org.apache.polaris.immutables.PolarisImmutable;
+
+/** Represents the local node's ID and informative, mutable state information. */
+@PolarisImmutable
+public interface Node {
+  /**
+   * Returns the ID of this node.
+   *
+   * @return ID of this node
+   * @throws IllegalStateException if the lease is no longer valid, for example, expired before it
+   *     being renewed
+   */
+  int id();
+
+  default boolean valid(long nowInMillis) {
+    return nowInMillis < expirationTimestamp().toEpochMilli();
+  }
+
+  Instant leaseTimestamp();
+
+  @Nullable
+  Instant renewLeaseTimestamp();
+
+  /** Timestamp since which this node's lease is no longer valid. */
+  Instant expirationTimestamp();
+}

persistence/nosql/nodes/api/src/main/java/org/apache/polaris/nodes/api/NodeLease.java

@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.nodes.api;
+
+import jakarta.annotation.Nullable;
+
+public interface NodeLease {
+  /**
+   * Returns the {@link Node} representation for this lease if the lease has not been released or
+   * {@code null}.
+   */
+  @Nullable
+  Node node();
+
+  /**
+   * Permanently release the lease. Does nothing, if already released. Throws if persisting the
+   * released state fails.
+   */
+  void release();
+
+  /**
+   * Force a lease renewal, generally not recommended nor necessary. Throws, if the lease is already
+   * released.
+   */
+  void renew();
+
+  /** Returns the node ID if the lease is active/valid or {@code -1}. */
+  int nodeIdIfValid();
+}

persistence/nosql/nodes/api/src/main/java/org/apache/polaris/nodes/api/NodeManagement.java

@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.nodes.api;
+
+import jakarta.annotation.Nonnull;
+import jakarta.enterprise.context.ApplicationScoped;
+import java.util.Optional;
+import org.apache.polaris.ids.api.IdGenerator;
+import org.apache.polaris.ids.api.SnowflakeIdGenerator;
+
+/**
+ * API to lease node IDs, primarily to generate {@linkplain SnowflakeIdGenerator snowflake IDs}.
+ *
+ * <p>The default configuration for the snowflake IDs allows generation of 4096 IDs per millisecond
+ * (12 sequence bits), which should be more than enough. As a consequence, it is very likely enough
+ * to have only one ID generator per JVM, across all realms and catalogs.
+ *
+ * <p>Implementation is provided as an {@link ApplicationScoped @ApplicationScoped} bean
+ */
+public interface NodeManagement extends AutoCloseable {
+  /**
+   * Build a <em>new</em> and <em>independent</em> ID generator instance of a {@linkplain #lease()
+   * leased node} using the given clock.
+   *
+   * <p>This function must only be called from {@link ApplicationScoped @ApplicationScoped} CDI
+   * producers providing the same {@link IdGenerator} for the lifetime of the given {@link Node},
+   * aka at most once for a {@link Node} instance.
+   */
+  IdGenerator buildIdGenerator(@Nonnull NodeLease leasedNode);
+
+  /** The maximum number of concurrently leased nodes that are supported. */
+  int maxNumberOfNodes();
+
+  /** Retrieve information about a specific node. */
+  Optional<Node> getNodeInfo(int nodeId);
+
+  /** Get the persistence ID for a node by its ID. */
+  long systemIdForNode(int nodeId);
+
+  /**
+   * Lease a node.
+   *
+   * <p>The implementation takes care of periodically renewing the lease. It is not necessary to
+   * explicitly call {@link NodeLease#renew()}.
+   *
+   * <p>It is possible that the {@linkplain NodeLease#nodeIdIfValid() ID of the leased node} changes
+   * over time.
+   *
+   * <p>Each invocation returns a new, independent lease.
+   *
+   * @return the leased node
+   * @throws IllegalStateException if no node ID could be leased
+   */
+  @Nonnull
+  NodeLease lease();
+}

persistence/nosql/nodes/api/src/main/java/org/apache/polaris/nodes/api/NodeManagementConfig.java

@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.nodes.api;
+
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import io.smallrye.config.ConfigMapping;
+import io.smallrye.config.WithDefault;
+import java.time.Duration;
+import org.apache.polaris.ids.api.IdGenerator;
+import org.apache.polaris.ids.api.IdGeneratorSpec;
+import org.apache.polaris.ids.api.SnowflakeIdGenerator;
+import org.apache.polaris.immutables.PolarisImmutable;
+import org.immutables.value.Value;
+
+/** Node management configuration. */
+@ConfigMapping(prefix = "polaris.node")
+@JsonSerialize(as = ImmutableBuildableNodeManagementConfig.class)
+@JsonDeserialize(as = ImmutableBuildableNodeManagementConfig.class)
+public interface NodeManagementConfig {
+  /** Duration of a node-lease. */
+  @WithDefault(DEFAULT_LEASE_DURATION)
+  Duration leaseDuration();
+
+  /** Time window before the end of a node lease when the lease will be renewed. */
+  @WithDefault(DEFAULT_RENEWAL_PERIOD)
+  Duration renewalPeriod();
+
+  /**
+   * Maximum number of concurrently active Polaris nodes. Do not change this value or the ID
+   * generator spec, it is a rather internal property. See ID generator spec below.
+   */
+  @WithDefault(DEFAULT_NUM_NODES)
+  int numNodes();
+
+  /**
+   * Configuration needed to build an {@linkplain IdGenerator ID generator}. This configuration
+   * cannot be changed after one Polaris node has been successfully started. This specification will
+   * be ignored if a persisted one exists, but a warning shall be logged if both are different.
+   */
+  IdGeneratorSpec idGeneratorSpec();
+
+  String DEFAULT_LEASE_DURATION = "PT1H";
+  String DEFAULT_RENEWAL_PERIOD = "PT15M";
+  String DEFAULT_NUM_NODES = "" + (1 << SnowflakeIdGenerator.DEFAULT_NODE_ID_BITS);
+
+  @PolarisImmutable
+  interface BuildableNodeManagementConfig extends NodeManagementConfig {
+
+    static ImmutableBuildableNodeManagementConfig.Builder builder() {
+      return ImmutableBuildableNodeManagementConfig.builder();
+    }
+
+    @Override
+    @Value.Default
+    default Duration leaseDuration() {
+      return Duration.parse(DEFAULT_LEASE_DURATION);
+    }
+
+    @Override
+    @Value.Default
+    default Duration renewalPeriod() {
+      return Duration.parse(DEFAULT_RENEWAL_PERIOD);
+    }
+
+    @Override
+    default int numNodes() {
+      return 1 << SnowflakeIdGenerator.DEFAULT_NODE_ID_BITS;
+    }
+  }
+}