From f0ca27b9ceb6a4f963cfda7c8a9a8329d9b0a235 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 27 Nov 2024 01:43:06 -0800 Subject: [PATCH 01/14] initial commit --- .../core/persistence/cache/EntityCache.java | 7 +- .../core/persistence/cache/EntityWeigher.java | 64 +++++++++++++++++++ .../{ => cache}/EntityCacheTest.java | 40 ++++++++++-- .../PolarisTestMetaStoreManager.java | 12 ++-- 4 files changed, 110 insertions(+), 13 deletions(-) create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityWeigher.java rename polaris-core/src/test/java/org/apache/polaris/core/persistence/{ => cache}/EntityCacheTest.java (93%) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCache.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCache.java index 3966b1e364..0f246fc468 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCache.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCache.java @@ -25,6 +25,8 @@ import java.util.List; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; + +import com.github.benmanes.caffeine.cache.Weigher; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisEntityType; @@ -71,12 +73,13 @@ public EntityCache(@NotNull PolarisRemoteCache polarisRemoteCache) { }; // use a Caffeine cache to purge entries when those have not been used for a long time. - // Assuming 1KB per entry, 100K entries is about 100MB. this.byId = Caffeine.newBuilder() - .maximumSize(100_000) // Set maximum size to 100,000 elements + .maximumWeight(100 * EntityWeigher.WEIGHT_PER_MB) // Goal is ~100MB + .weigher(EntityWeigher.asWeigher()) .expireAfterAccess(1, TimeUnit.HOURS) // Expire entries after 1 hour of no access .removalListener(removalListener) // Set the removal listener + .softValues() // Account for memory pressure .build(); // remember the meta store manager diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityWeigher.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityWeigher.java new file mode 100644 index 0000000000..0efe295a0a --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityWeigher.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.core.persistence.cache; + +import com.github.benmanes.caffeine.cache.Weigher; +import org.checkerframework.checker.index.qual.NonNegative; + +/** + * A {@link Weigher} implementation that weighs {@link EntityCacheEntry} objects + * by the approximate size of the entity object. + */ +public class EntityWeigher implements Weigher { + + /** The amount of weight that is expected to roughly equate to 1MB of memory usage */ + public static final long WEIGHT_PER_MB = 1024 * 1024; + + /* Represents the approximate size of an entity beyond the properties */ + private static final int APPROXIMATE_ENTITY_OVERHEAD = 1000; + + /** Singleton instance */ + private static final EntityWeigher instance = new EntityWeigher(); + + private EntityWeigher() {} + + /** Gets the singleton {@link EntityWeigher} */ + public static EntityWeigher getInstance() { + return instance; + } + + /** + * Computes the weight of a given entity + * @param key The entity's key; not used + * @param value The entity to be cached + * @return The weight of the entity + */ + @Override + public @NonNegative int weigh(Long key, EntityCacheEntry value) { + return APPROXIMATE_ENTITY_OVERHEAD + + value.getEntity().getProperties().length() + + value.getEntity().getInternalProperties().length(); + } + + /** Factory method to provide a typed Weigher */ + public static Weigher asWeigher() { + return (Weigher) getInstance(); + } +} diff --git a/polaris-core/src/test/java/org/apache/polaris/core/persistence/EntityCacheTest.java b/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/EntityCacheTest.java similarity index 93% rename from polaris-core/src/test/java/org/apache/polaris/core/persistence/EntityCacheTest.java rename to polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/EntityCacheTest.java index c47367769e..cccaaa8e2e 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/persistence/EntityCacheTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/EntityCacheTest.java @@ -16,24 +16,30 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.core.persistence; +package org.apache.polaris.core.persistence.cache; import static org.apache.polaris.core.persistence.PrincipalSecretsGenerator.RANDOM_SECRETS; import java.util.List; import java.util.stream.Collectors; + +import org.apache.iceberg.catalog.TableIdentifier; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisEntity; import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PolarisGrantRecord; import org.apache.polaris.core.entity.PolarisPrivilege; -import org.apache.polaris.core.persistence.cache.EntityCache; -import org.apache.polaris.core.persistence.cache.EntityCacheByNameKey; -import org.apache.polaris.core.persistence.cache.EntityCacheEntry; -import org.apache.polaris.core.persistence.cache.EntityCacheLookupResult; +import org.apache.polaris.core.entity.TableLikeEntity; +import org.apache.polaris.core.persistence.PolarisMetaStoreManager; +import org.apache.polaris.core.persistence.PolarisMetaStoreManagerImpl; +import org.apache.polaris.core.persistence.PolarisMetaStoreSession; +import org.apache.polaris.core.persistence.PolarisTestMetaStoreManager; +import org.apache.polaris.core.persistence.PolarisTreeMapMetaStoreSessionImpl; +import org.apache.polaris.core.persistence.PolarisTreeMapStore; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; import org.mockito.Mockito; @@ -473,4 +479,28 @@ void testRenameAndCacheDestinationBeforeLoadingSource() { // now the loading by the old name should return null Assertions.assertThat(cache.getOrLoadEntityByName(callCtx, T4_name)).isNull(); } + + /* Helper for `testEntityWeigher` */ + private int getEntityWeight(PolarisEntity entity) { + return EntityWeigher.getInstance() + .weigh( + -1L, + new EntityCacheEntry(diagServices, 1L, entity, List.of(), 1)); + } + + @Test + void testEntityWeigher() { + var smallEntity = new TableLikeEntity.Builder(TableIdentifier.of("ns.t1"), "").build(); + var mediumEntity = + new TableLikeEntity.Builder(TableIdentifier.of("ns.t1"), "") + .setMetadataLocation("a".repeat(10000)) + .build(); + var largeEntity = + new TableLikeEntity.Builder(TableIdentifier.of("ns.t1"), "") + .setMetadataLocation("a".repeat(1000 * 1000)) + .build(); + + Assertions.assertThat(getEntityWeight(smallEntity)).isLessThan(getEntityWeight(mediumEntity)); + Assertions.assertThat(getEntityWeight(mediumEntity)).isLessThan(getEntityWeight(largeEntity)); + } } diff --git a/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/PolarisTestMetaStoreManager.java b/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/PolarisTestMetaStoreManager.java index 53b0e8408f..7377e4e5ae 100644 --- a/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/PolarisTestMetaStoreManager.java +++ b/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/PolarisTestMetaStoreManager.java @@ -828,7 +828,7 @@ void dropEntity(List catalogPath, PolarisEntityCore entityToD } /** Grant a privilege to a catalog role */ - void grantPrivilege( + public void grantPrivilege( PolarisBaseEntity role, List catalogPath, PolarisBaseEntity securable, @@ -1006,7 +1006,7 @@ PolarisBaseEntity createTestCatalog(String catalogName) { * * @return the identity we found */ - PolarisBaseEntity ensureExistsByName( + public PolarisBaseEntity ensureExistsByName( List catalogPath, PolarisEntityType entityType, PolarisEntitySubType entitySubType, @@ -1052,7 +1052,7 @@ PolarisBaseEntity ensureExistsByName( * * @return the identity we found */ - PolarisBaseEntity ensureExistsByName( + public PolarisBaseEntity ensureExistsByName( List catalogPath, PolarisEntityType entityType, String name) { return this.ensureExistsByName( catalogPath, entityType, PolarisEntitySubType.NULL_SUBTYPE, name); @@ -1067,7 +1067,7 @@ PolarisBaseEntity ensureExistsByName( * @param internalProps updated internal properties * @return updated entity */ - PolarisBaseEntity updateEntity( + public PolarisBaseEntity updateEntity( List catalogPath, PolarisBaseEntity entity, String props, @@ -1553,7 +1553,7 @@ void validateBootstrap() { this.ensureGrantRecordExists(principalRole, principal, PolarisPrivilege.PRINCIPAL_ROLE_USAGE); } - void testCreateTestCatalog() { + public void testCreateTestCatalog() { // create test catalog this.createTestCatalog("test"); @@ -2075,7 +2075,7 @@ public void testPrivileges() { * @param newCatPath new catalog path * @param newName new name */ - void renameEntity( + public void renameEntity( List catPath, PolarisBaseEntity entity, List newCatPath, From e28c5198f3f782289c5f69492629ea0352ac806a Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 27 Nov 2024 01:43:15 -0800 Subject: [PATCH 02/14] autolint --- .../core/persistence/cache/EntityCache.java | 2 - .../core/persistence/cache/EntityWeigher.java | 60 +++++++++---------- .../persistence/cache/EntityCacheTest.java | 11 ++-- 3 files changed, 34 insertions(+), 39 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCache.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCache.java index 0f246fc468..402aec9f89 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCache.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCache.java @@ -25,8 +25,6 @@ import java.util.List; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; - -import com.github.benmanes.caffeine.cache.Weigher; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisEntityType; diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityWeigher.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityWeigher.java index 0efe295a0a..bc937c0e93 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityWeigher.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityWeigher.java @@ -16,49 +16,49 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.polaris.core.persistence.cache; import com.github.benmanes.caffeine.cache.Weigher; import org.checkerframework.checker.index.qual.NonNegative; /** - * A {@link Weigher} implementation that weighs {@link EntityCacheEntry} objects - * by the approximate size of the entity object. + * A {@link Weigher} implementation that weighs {@link EntityCacheEntry} objects by the approximate + * size of the entity object. */ public class EntityWeigher implements Weigher { - /** The amount of weight that is expected to roughly equate to 1MB of memory usage */ - public static final long WEIGHT_PER_MB = 1024 * 1024; + /** The amount of weight that is expected to roughly equate to 1MB of memory usage */ + public static final long WEIGHT_PER_MB = 1024 * 1024; - /* Represents the approximate size of an entity beyond the properties */ - private static final int APPROXIMATE_ENTITY_OVERHEAD = 1000; + /* Represents the approximate size of an entity beyond the properties */ + private static final int APPROXIMATE_ENTITY_OVERHEAD = 1000; - /** Singleton instance */ - private static final EntityWeigher instance = new EntityWeigher(); + /** Singleton instance */ + private static final EntityWeigher instance = new EntityWeigher(); - private EntityWeigher() {} + private EntityWeigher() {} - /** Gets the singleton {@link EntityWeigher} */ - public static EntityWeigher getInstance() { - return instance; - } + /** Gets the singleton {@link EntityWeigher} */ + public static EntityWeigher getInstance() { + return instance; + } - /** - * Computes the weight of a given entity - * @param key The entity's key; not used - * @param value The entity to be cached - * @return The weight of the entity - */ - @Override - public @NonNegative int weigh(Long key, EntityCacheEntry value) { - return APPROXIMATE_ENTITY_OVERHEAD + - value.getEntity().getProperties().length() + - value.getEntity().getInternalProperties().length(); - } + /** + * Computes the weight of a given entity + * + * @param key The entity's key; not used + * @param value The entity to be cached + * @return The weight of the entity + */ + @Override + public @NonNegative int weigh(Long key, EntityCacheEntry value) { + return APPROXIMATE_ENTITY_OVERHEAD + + value.getEntity().getProperties().length() + + value.getEntity().getInternalProperties().length(); + } - /** Factory method to provide a typed Weigher */ - public static Weigher asWeigher() { - return (Weigher) getInstance(); - } + /** Factory method to provide a typed Weigher */ + public static Weigher asWeigher() { + return (Weigher) getInstance(); + } } diff --git a/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/EntityCacheTest.java b/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/EntityCacheTest.java index cccaaa8e2e..a2af61b934 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/EntityCacheTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/EntityCacheTest.java @@ -22,7 +22,6 @@ import java.util.List; import java.util.stream.Collectors; - import org.apache.iceberg.catalog.TableIdentifier; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; @@ -483,16 +482,14 @@ void testRenameAndCacheDestinationBeforeLoadingSource() { /* Helper for `testEntityWeigher` */ private int getEntityWeight(PolarisEntity entity) { return EntityWeigher.getInstance() - .weigh( - -1L, - new EntityCacheEntry(diagServices, 1L, entity, List.of(), 1)); + .weigh(-1L, new EntityCacheEntry(diagServices, 1L, entity, List.of(), 1)); } @Test void testEntityWeigher() { - var smallEntity = new TableLikeEntity.Builder(TableIdentifier.of("ns.t1"), "").build(); - var mediumEntity = - new TableLikeEntity.Builder(TableIdentifier.of("ns.t1"), "") + var smallEntity = new TableLikeEntity.Builder(TableIdentifier.of("ns.t1"), "").build(); + var mediumEntity = + new TableLikeEntity.Builder(TableIdentifier.of("ns.t1"), "") .setMetadataLocation("a".repeat(10000)) .build(); var largeEntity = From fd351a175fc268593573593e5b1336803dca184e Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Thu, 23 Jan 2025 12:47:41 -0800 Subject: [PATCH 03/14] resolve conflicts --- .../polaris/core/persistence/cache/EntityCacheTest.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/EntityCacheTest.java b/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/EntityCacheTest.java index fa1131f458..cc32702136 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/EntityCacheTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/EntityCacheTest.java @@ -23,12 +23,8 @@ import java.time.Clock; import java.util.List; import java.util.stream.Collectors; -<<<<<<< HEAD:polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/EntityCacheTest.java import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.polaris.core.PolarisCallContext; -======= import org.apache.polaris.core.PolarisConfigurationStore; ->>>>>>> 012acdd10a0c0c54b863f3225027726b1d04598a:polaris-core/src/test/java/org/apache/polaris/core/persistence/EntityCacheTest.java import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.context.RealmId; From 93a5de182fdd235f45cb5e587af4a561b73aef39 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Thu, 13 Mar 2025 21:09:53 -0700 Subject: [PATCH 04/14] autolint --- .../core/config/BehaviorChangeConfiguration.java | 3 +-- .../core/persistence/cache/EntityCache.java | 14 +++++++------- .../core/persistence/cache/EntityWeigher.java | 4 ++-- .../core/persistence/cache/EntityCacheTest.java | 3 --- 4 files changed, 10 insertions(+), 14 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/config/BehaviorChangeConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/config/BehaviorChangeConfiguration.java index dd66ba31e7..91038c8cc6 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/config/BehaviorChangeConfiguration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/config/BehaviorChangeConfiguration.java @@ -56,8 +56,7 @@ protected BehaviorChangeConfiguration( public static final BehaviorChangeConfiguration ENTITY_CACHE_SOFT_VALUES = PolarisConfiguration.builder() .key("ENTITY_CACHE_SOFT_VALUES") - .description( - "Whether or not to use soft values in the entity cache") + .description("Whether or not to use soft values in the entity cache") .defaultValue(false) .buildBehaviorChangeConfiguration(); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCache.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCache.java index 8291ed9447..8d6357212a 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCache.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCache.java @@ -30,7 +30,6 @@ import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.config.BehaviorChangeConfiguration; import org.apache.polaris.core.config.PolarisConfiguration; -import org.apache.polaris.core.config.PolarisConfigurationStore; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PolarisGrantRecord; @@ -75,13 +74,14 @@ public EntityCache(@Nonnull PolarisMetaStoreManager polarisMetaStoreManager) { } }; - Caffeine byIdBuilder = Caffeine.newBuilder() - .maximumWeight(100 * EntityWeigher.WEIGHT_PER_MB) // Goal is ~100MB - .weigher(EntityWeigher.asWeigher()) - .expireAfterAccess(1, TimeUnit.HOURS) // Expire entries after 1 hour of no access - .removalListener(removalListener); // Set the removal listener + Caffeine byIdBuilder = + Caffeine.newBuilder() + .maximumWeight(100 * EntityWeigher.WEIGHT_PER_MB) // Goal is ~100MB + .weigher(EntityWeigher.asWeigher()) + .expireAfterAccess(1, TimeUnit.HOURS) // Expire entries after 1 hour of no access + .removalListener(removalListener); // Set the removal listener - if ( PolarisConfiguration.loadConfig(BehaviorChangeConfiguration.ENTITY_CACHE_SOFT_VALUES)) { + if (PolarisConfiguration.loadConfig(BehaviorChangeConfiguration.ENTITY_CACHE_SOFT_VALUES)) { byIdBuilder.softValues(); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityWeigher.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityWeigher.java index 16266a28d8..1e184356a3 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityWeigher.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityWeigher.java @@ -23,8 +23,8 @@ import org.checkerframework.checker.index.qual.NonNegative; /** - * A {@link Weigher} implementation that weighs {@link ResolvedPolarisEntity} objects by the approximate - * size of the entity object. + * A {@link Weigher} implementation that weighs {@link ResolvedPolarisEntity} objects by the + * approximate size of the entity object. */ public class EntityWeigher implements Weigher { diff --git a/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/EntityCacheTest.java b/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/EntityCacheTest.java index f8c120303e..dac8815de8 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/EntityCacheTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/EntityCacheTest.java @@ -36,9 +36,6 @@ import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.PolarisTestMetaStoreManager; import org.apache.polaris.core.persistence.ResolvedPolarisEntity; -import org.apache.polaris.core.persistence.cache.EntityCache; -import org.apache.polaris.core.persistence.cache.EntityCacheByNameKey; -import org.apache.polaris.core.persistence.cache.EntityCacheLookupResult; import org.apache.polaris.core.persistence.transactional.PolarisMetaStoreManagerImpl; import org.apache.polaris.core.persistence.transactional.PolarisTreeMapMetaStoreSessionImpl; import org.apache.polaris.core.persistence.transactional.PolarisTreeMapStore; From 4e9d19274bb1121a86a71eeb99b7c0d47be1eed2 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Fri, 14 Mar 2025 11:31:21 -0700 Subject: [PATCH 05/14] pull main --- .../java/org/apache/polaris/service/TestServices.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java b/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java index 0f63106ac2..d9520b59aa 100644 --- a/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java +++ b/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java @@ -132,10 +132,6 @@ public TestServices build() { RealmEntityManagerFactory realmEntityManagerFactory = new RealmEntityManagerFactory(metaStoreManagerFactory) {}; - PolarisEntityManager entityManager = - realmEntityManagerFactory.getOrCreateEntityManager(realmContext); - PolarisMetaStoreManager metaStoreManager = - metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext); TransactionalPersistence metaStoreSession = metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(); CallContext callContext = @@ -159,6 +155,11 @@ public Map contextVariables() { return new HashMap<>(); } }; + CallContext.setCurrentContext(callContext); + PolarisEntityManager entityManager = + realmEntityManagerFactory.getOrCreateEntityManager(realmContext); + PolarisMetaStoreManager metaStoreManager = + metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext); FileIOFactory fileIOFactory = fileIOFactorySupplier.apply( From ce7b2e56a3e4344ca3ce2e215e5c346c7ef94c5a Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Fri, 14 Mar 2025 15:00:37 -0700 Subject: [PATCH 06/14] Add multiplier --- .../polaris/core/persistence/cache/EntityWeigher.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityWeigher.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityWeigher.java index 1e184356a3..5257f6d649 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityWeigher.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityWeigher.java @@ -34,6 +34,9 @@ public class EntityWeigher implements Weigher { /* Represents the approximate size of an entity beyond the properties */ private static final int APPROXIMATE_ENTITY_OVERHEAD = 1000; + /* Represents the amount of bytes that a character is expected to take up */ + private static final int APPROXIMATE_BYTES_PER_CHAR = 2; + /** Singleton instance */ private static final EntityWeigher instance = new EntityWeigher(); @@ -45,7 +48,7 @@ public static EntityWeigher getInstance() { } /** - * Computes the weight of a given entity + * Computes the weight of a given entity. The unit here is not exactly bytes, but it's close. * * @param key The entity's key; not used * @param value The entity to be cached @@ -54,8 +57,8 @@ public static EntityWeigher getInstance() { @Override public @NonNegative int weigh(Long key, ResolvedPolarisEntity value) { return APPROXIMATE_ENTITY_OVERHEAD - + value.getEntity().getProperties().length() - + value.getEntity().getInternalProperties().length(); + + (value.getEntity().getProperties().length() * APPROXIMATE_BYTES_PER_CHAR) + + (value.getEntity().getInternalProperties().length() * APPROXIMATE_BYTES_PER_CHAR); } /** Factory method to provide a typed Weigher */ From ed42b8c9b6477e1802a0f890a2fd04e0023b4843 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Fri, 14 Mar 2025 15:02:38 -0700 Subject: [PATCH 07/14] account for name, too --- .../org/apache/polaris/core/persistence/cache/EntityWeigher.java | 1 + 1 file changed, 1 insertion(+) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityWeigher.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityWeigher.java index 5257f6d649..a6050dea8f 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityWeigher.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityWeigher.java @@ -57,6 +57,7 @@ public static EntityWeigher getInstance() { @Override public @NonNegative int weigh(Long key, ResolvedPolarisEntity value) { return APPROXIMATE_ENTITY_OVERHEAD + + (value.getEntity().getName().length() * APPROXIMATE_BYTES_PER_CHAR) + (value.getEntity().getProperties().length() * APPROXIMATE_BYTES_PER_CHAR) + (value.getEntity().getInternalProperties().length() * APPROXIMATE_BYTES_PER_CHAR); } From 94c3fe327959f1b4a063718bbe5894cac385409b Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Fri, 14 Mar 2025 15:15:28 -0700 Subject: [PATCH 08/14] adjust multiplier --- .../apache/polaris/core/persistence/cache/EntityWeigher.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityWeigher.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityWeigher.java index a6050dea8f..f6c4385a72 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityWeigher.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityWeigher.java @@ -35,7 +35,7 @@ public class EntityWeigher implements Weigher { private static final int APPROXIMATE_ENTITY_OVERHEAD = 1000; /* Represents the amount of bytes that a character is expected to take up */ - private static final int APPROXIMATE_BYTES_PER_CHAR = 2; + private static final int APPROXIMATE_BYTES_PER_CHAR = 3; /** Singleton instance */ private static final EntityWeigher instance = new EntityWeigher(); From 1a26018ae4e05f875a6f82b1f5a69fa262da1069 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Fri, 14 Mar 2025 15:18:01 -0700 Subject: [PATCH 09/14] add config --- .../apache/polaris/core/config/FeatureConfiguration.java | 8 ++++++++ .../polaris/core/persistence/cache/EntityCache.java | 4 +++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java index 087d0525ee..7f12899337 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java @@ -21,6 +21,7 @@ import java.util.List; import java.util.Optional; import org.apache.polaris.core.admin.model.StorageConfigInfo; +import org.apache.polaris.core.persistence.cache.EntityWeigher; /** * Configurations for features within Polaris. These configurations are intended to be customized @@ -183,4 +184,11 @@ protected FeatureConfiguration( "How many times to retry refreshing metadata when the previous error was retryable") .defaultValue(2) .buildFeatureConfiguration(); + + public static final FeatureConfiguration ENTITY_CACHE_WEIGHER_TARGET = + PolarisConfiguration.builder() + .key("ENTITY_CACHE_WEIGHER_TARGET") + .description("The target weight for the entity cache.") + .defaultValue(100 * EntityWeigher.WEIGHT_PER_MB) + .buildFeatureConfiguration(); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCache.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCache.java index 8d6357212a..a56238601e 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCache.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCache.java @@ -29,6 +29,7 @@ import java.util.concurrent.TimeUnit; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.config.BehaviorChangeConfiguration; +import org.apache.polaris.core.config.FeatureConfiguration; import org.apache.polaris.core.config.PolarisConfiguration; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisEntityType; @@ -74,9 +75,10 @@ public EntityCache(@Nonnull PolarisMetaStoreManager polarisMetaStoreManager) { } }; + long weigherTarget = PolarisConfiguration.loadConfig(FeatureConfiguration.ENTITY_CACHE_WEIGHER_TARGET); Caffeine byIdBuilder = Caffeine.newBuilder() - .maximumWeight(100 * EntityWeigher.WEIGHT_PER_MB) // Goal is ~100MB + .maximumWeight(weigherTarget) .weigher(EntityWeigher.asWeigher()) .expireAfterAccess(1, TimeUnit.HOURS) // Expire entries after 1 hour of no access .removalListener(removalListener); // Set the removal listener From 230c95eb1a84b106f55cf873e02a9fff97e20310 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Fri, 14 Mar 2025 15:18:03 -0700 Subject: [PATCH 10/14] autolint --- .../org/apache/polaris/core/persistence/cache/EntityCache.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCache.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCache.java index a56238601e..752342ffe6 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCache.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCache.java @@ -75,7 +75,8 @@ public EntityCache(@Nonnull PolarisMetaStoreManager polarisMetaStoreManager) { } }; - long weigherTarget = PolarisConfiguration.loadConfig(FeatureConfiguration.ENTITY_CACHE_WEIGHER_TARGET); + long weigherTarget = + PolarisConfiguration.loadConfig(FeatureConfiguration.ENTITY_CACHE_WEIGHER_TARGET); Caffeine byIdBuilder = Caffeine.newBuilder() .maximumWeight(weigherTarget) From eb0a8179516514811b2a576d8351d7e39ffbe8e9 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 17 Mar 2025 21:56:23 -0700 Subject: [PATCH 11/14] remove old cast --- .../apache/polaris/core/persistence/cache/EntityWeigher.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityWeigher.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityWeigher.java index f6c4385a72..b3409d3b90 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityWeigher.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityWeigher.java @@ -64,6 +64,6 @@ public static EntityWeigher getInstance() { /** Factory method to provide a typed Weigher */ public static Weigher asWeigher() { - return (Weigher) getInstance(); + return getInstance(); } } From fc9daeeef5c1519aaa76bafd54405e9b7bb536d8 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Tue, 8 Apr 2025 11:01:16 -0700 Subject: [PATCH 12/14] more tests, fixes per review --- .../core/config/FeatureConfiguration.java | 5 +- .../persistence/cache/EntityWeigherTest.java | 101 ++++++++++++++++++ 2 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/EntityWeigherTest.java diff --git a/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java index 40bcd29013..6be57f178b 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java @@ -195,7 +195,10 @@ protected FeatureConfiguration( public static final FeatureConfiguration ENTITY_CACHE_WEIGHER_TARGET = PolarisConfiguration.builder() .key("ENTITY_CACHE_WEIGHER_TARGET") - .description("The target weight for the entity cache.") + .description( + "The maximum weight for the entity cache. This is a heuristic value without any particular" + + " unit of measurement. It roughly correlates with the total heap size of cached values. Fine-tuning" + + " requires experimentation in the specific deployment environment") .defaultValue(100 * EntityWeigher.WEIGHT_PER_MB) .buildFeatureConfiguration(); } diff --git a/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/EntityWeigherTest.java b/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/EntityWeigherTest.java new file mode 100644 index 0000000000..59c0aed34e --- /dev/null +++ b/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/EntityWeigherTest.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.persistence.cache; + +import java.util.List; +import java.util.Optional; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; +import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.entity.table.IcebergTableLikeEntity; +import org.apache.polaris.core.persistence.ResolvedPolarisEntity; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; + +public class EntityWeigherTest { + + private PolarisDiagnostics diagnostics; + + public EntityWeigherTest() { + diagnostics = new PolarisDefaultDiagServiceImpl(); + } + + private ResolvedPolarisEntity getEntity( + String name, + String metadataLocation, + String properties, + Optional internalProperties) { + var entity = + new IcebergTableLikeEntity.Builder(TableIdentifier.of(name), metadataLocation).build(); + entity.setProperties(properties); + internalProperties.ifPresent(p -> entity.setInternalProperties(p)); + return new ResolvedPolarisEntity(diagnostics, entity, List.of(), 1); + } + + @Test + public void testBasicWeight() { + int weight = EntityWeigher.getInstance().weigh(1L, getEntity("t", "", "", Optional.empty())); + Assertions.assertThat(weight).isGreaterThan(0); + } + + @Test + public void testNonZeroWeight() { + int weight = EntityWeigher.getInstance().weigh(1L, getEntity("t", "", "", Optional.of(""))); + Assertions.assertThat(weight).isGreaterThan(0); + } + + @Test + public void testWeightIncreasesWithNameLength() { + int smallWeight = + EntityWeigher.getInstance().weigh(1L, getEntity("t", "", "", Optional.empty())); + int largeWeight = + EntityWeigher.getInstance().weigh(1L, getEntity("looong name", "", "", Optional.empty())); + Assertions.assertThat(smallWeight).isLessThan(largeWeight); + } + + @Test + public void testWeightIncreasesWithMetadataLocationLength() { + int smallWeight = + EntityWeigher.getInstance().weigh(1L, getEntity("t", "", "", Optional.empty())); + int largeWeight = + EntityWeigher.getInstance() + .weigh(1L, getEntity("t", "looong location", "", Optional.empty())); + Assertions.assertThat(smallWeight).isLessThan(largeWeight); + } + + @Test + public void testWeightIncreasesWithPropertiesLength() { + int smallWeight = + EntityWeigher.getInstance().weigh(1L, getEntity("t", "", "", Optional.empty())); + int largeWeight = + EntityWeigher.getInstance() + .weigh(1L, getEntity("t", "", "looong properties", Optional.empty())); + Assertions.assertThat(smallWeight).isLessThan(largeWeight); + } + + @Test + public void testWeightIncreasesWithInternalPropertiesLength() { + int smallWeight = + EntityWeigher.getInstance().weigh(1L, getEntity("t", "", "", Optional.of(""))); + int largeWeight = + EntityWeigher.getInstance() + .weigh(1L, getEntity("t", "", "", Optional.of("looong iproperties"))); + Assertions.assertThat(smallWeight).isLessThan(largeWeight); + } +} From f399043fee71504f8b3a5bfea3e8eed610d7c2a0 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Tue, 8 Apr 2025 15:24:47 -0700 Subject: [PATCH 13/14] add precise weight test --- .../polaris/core/persistence/cache/EntityWeigherTest.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/EntityWeigherTest.java b/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/EntityWeigherTest.java index 59c0aed34e..2f30363591 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/EntityWeigherTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/EntityWeigherTest.java @@ -98,4 +98,12 @@ public void testWeightIncreasesWithInternalPropertiesLength() { .weigh(1L, getEntity("t", "", "", Optional.of("looong iproperties"))); Assertions.assertThat(smallWeight).isLessThan(largeWeight); } + + @Test + public void testExactWeightCalculation() { + int preciseWeight = + EntityWeigher.getInstance().weigh(1L, + getEntity("name", "location", "{a: b}", Optional.of("{c: d, e: f}"))); + Assertions.assertThat(preciseWeight).isEqualTo(1066); + } } From ca620aa33d6bf0abf5400386ec676659ef6d415a Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Tue, 8 Apr 2025 15:24:50 -0700 Subject: [PATCH 14/14] autolint --- .../core/persistence/cache/EntityWeigherTest.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/EntityWeigherTest.java b/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/EntityWeigherTest.java index 2f30363591..e989329c5c 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/EntityWeigherTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/EntityWeigherTest.java @@ -99,11 +99,11 @@ public void testWeightIncreasesWithInternalPropertiesLength() { Assertions.assertThat(smallWeight).isLessThan(largeWeight); } - @Test - public void testExactWeightCalculation() { - int preciseWeight = - EntityWeigher.getInstance().weigh(1L, - getEntity("name", "location", "{a: b}", Optional.of("{c: d, e: f}"))); - Assertions.assertThat(preciseWeight).isEqualTo(1066); - } + @Test + public void testExactWeightCalculation() { + int preciseWeight = + EntityWeigher.getInstance() + .weigh(1L, getEntity("name", "location", "{a: b}", Optional.of("{c: d, e: f}"))); + Assertions.assertThat(preciseWeight).isEqualTo(1066); + } }