Skip to content

Commit a7f33c1

Browse files
adutraadutra
committed
Upgrade Polaris to Iceberg 1.9.0
1 parent b9e3c15 commit a7f33c1

File tree

11 files changed

+156
-50
lines changed

11 files changed

+156
-50
lines changed

‎gradle/libs.versions.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919

2020
[versions]
2121
hadoop = "3.4.1"
22-
iceberg = "1.8.1" # Ensure to update the iceberg version in regtests to keep regtests up-to-date
22+
iceberg = "1.9.0" # Ensure to update the iceberg version in regtests to keep regtests up-to-date
2323
quarkus = "3.21.2"
2424
immutables = "2.10.1"
2525
picocli = "4.7.7"

‎integration-tests/src/main/java/org/apache/polaris/service/it/env/IcebergHelper.java‎

Lines changed: 1 addition & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,6 @@
2020

2121
import com.google.common.collect.ImmutableMap;
2222
import java.util.Map;
23-
import org.apache.iceberg.catalog.SessionCatalog;
24-
import org.apache.iceberg.rest.HTTPClient;
2523
import org.apache.iceberg.rest.RESTCatalog;
2624
import org.apache.iceberg.rest.auth.OAuth2Properties;
2725
import org.apache.polaris.core.admin.model.PrincipalWithCredentials;
@@ -36,14 +34,7 @@ public static RESTCatalog restCatalog(
3634
String catalog,
3735
Map<String, String> extraProperties) {
3836
String authToken = client.obtainToken(credentials);
39-
SessionCatalog.SessionContext context = SessionCatalog.SessionContext.createEmpty();
40-
RESTCatalog restCatalog =
41-
new RESTCatalog(
42-
context,
43-
(config) ->
44-
HTTPClient.builder(config)
45-
.uri(config.get(org.apache.iceberg.CatalogProperties.URI))
46-
.build());
37+
RESTCatalog restCatalog = new RESTCatalog();
4738

4839
ImmutableMap.Builder<String, String> propertiesBuilder =
4940
ImmutableMap.<String, String>builder()

‎integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java‎

Lines changed: 35 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -371,8 +371,11 @@ public void testIcebergCreateTablesInExternalCatalog() throws IOException {
371371
TableIdentifier.of(ns, "the_table"),
372372
new Schema(
373373
List.of(
374-
Types.NestedField.of(
375-
1, false, "theField", Types.StringType.get()))))
374+
Types.NestedField.builder()
375+
.withId(1)
376+
.withName("theField")
377+
.ofType(Types.StringType.get())
378+
.build())))
376379
.withLocation("file:///tmp/tables")
377380
.withSortOrder(SortOrder.unsorted())
378381
.withPartitionSpec(PartitionSpec.unpartitioned())
@@ -399,8 +402,11 @@ public void testIcebergCreateTablesWithWritePathBlocked() throws IOException {
399402
TableIdentifier.of(ns, "the_table"),
400403
new Schema(
401404
List.of(
402-
Types.NestedField.of(
403-
1, false, "theField", Types.StringType.get()))))
405+
Types.NestedField.builder()
406+
.withId(1)
407+
.withName("theField")
408+
.ofType(Types.StringType.get())
409+
.build())))
404410
.withSortOrder(SortOrder.unsorted())
405411
.withPartitionSpec(PartitionSpec.unpartitioned())
406412
.withProperties(Map.of("write.data.path", "s3://my-bucket/path/to/data"))
@@ -416,8 +422,11 @@ public void testIcebergCreateTablesWithWritePathBlocked() throws IOException {
416422
TableIdentifier.of(ns, "the_table"),
417423
new Schema(
418424
List.of(
419-
Types.NestedField.of(
420-
1, false, "theField", Types.StringType.get()))))
425+
Types.NestedField.builder()
426+
.withId(1)
427+
.withName("theField")
428+
.ofType(Types.StringType.get())
429+
.build())))
421430
.withSortOrder(SortOrder.unsorted())
422431
.withPartitionSpec(PartitionSpec.unpartitioned())
423432
.withProperties(Map.of("write.metadata.path", "s3://my-bucket/path/to/data"))
@@ -454,7 +463,13 @@ public void testIcebergRegisterTableInExternalCatalog() throws IOException {
454463
.assignUUID()
455464
.addPartitionSpec(PartitionSpec.unpartitioned())
456465
.addSortOrder(SortOrder.unsorted())
457-
.addSchema(new Schema(Types.NestedField.of(1, false, "col1", Types.StringType.get())))
466+
.addSchema(
467+
new Schema(
468+
Types.NestedField.builder()
469+
.withId(1)
470+
.withName("col1")
471+
.ofType(Types.StringType.get())
472+
.build()))
458473
.build();
459474
TableMetadataParser.write(tableMetadata, fileIo.newOutputFile(metadataLocation));
460475

@@ -490,7 +505,12 @@ public void testIcebergUpdateTableInExternalCatalog() throws IOException {
490505
String location = baseLocation.resolve("testIcebergUpdateTableInExternalCatalog").toString();
491506
String metadataLocation = location + "/metadata/000001-494949494949494949.metadata.json";
492507

493-
Types.NestedField col1 = Types.NestedField.of(1, false, "col1", Types.StringType.get());
508+
Types.NestedField col1 =
509+
Types.NestedField.builder()
510+
.withId(1)
511+
.withName("col1")
512+
.ofType(Types.StringType.get())
513+
.build();
494514
TableMetadata tableMetadata =
495515
TableMetadata.buildFromEmpty()
496516
.setLocation(location)
@@ -545,7 +565,13 @@ public void testIcebergDropTableInExternalCatalog() throws IOException {
545565
.assignUUID()
546566
.addPartitionSpec(PartitionSpec.unpartitioned())
547567
.addSortOrder(SortOrder.unsorted())
548-
.addSchema(new Schema(Types.NestedField.of(1, false, "col1", Types.StringType.get())))
568+
.addSchema(
569+
new Schema(
570+
Types.NestedField.builder()
571+
.withId(1)
572+
.withName("col1")
573+
.ofType(Types.StringType.get())
574+
.build()))
549575
.build();
550576
TableMetadataParser.write(tableMetadata, fileIo.newOutputFile(metadataLocation));
551577

‎integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java‎

Lines changed: 62 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,19 @@ public class PolarisRestCatalogIntegrationTest extends CatalogTests<RESTCatalog>
148148
private final String catalogBaseLocation =
149149
s3BucketBase + "/" + System.getenv("USER") + "/path/to/data";
150150

151+
private static final Map<String, String> DEFAULT_REST_CATALOG_CONFIG =
152+
Map.of(
153+
org.apache.iceberg.CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key1",
154+
"catalog-default-key1",
155+
org.apache.iceberg.CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key2",
156+
"catalog-default-key2",
157+
org.apache.iceberg.CatalogProperties.TABLE_DEFAULT_PREFIX + "override-key3",
158+
"catalog-default-key3",
159+
org.apache.iceberg.CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key3",
160+
"catalog-override-key3",
161+
org.apache.iceberg.CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key4",
162+
"catalog-override-key4");
163+
151164
private static final String[] DEFAULT_CATALOG_PROPERTIES = {
152165
"allow.unstructured.table.location", "true",
153166
"allow.external.table.location", "true"
@@ -235,7 +248,7 @@ public void before(TestInfo testInfo) {
235248

236249
managementApi.createCatalog(principalRoleName, catalog);
237250

238-
restCatalogConfig =
251+
Map<String, String> dynamicConfig =
239252
testInfo
240253
.getTestMethod()
241254
.map(m -> m.getAnnotation(RestCatalogConfig.class))
@@ -254,6 +267,12 @@ public void before(TestInfo testInfo) {
254267
})
255268
.orElse(ImmutableMap.of());
256269

270+
restCatalogConfig =
271+
ImmutableMap.<String, String>builder()
272+
.putAll(DEFAULT_REST_CATALOG_CONFIG)
273+
.putAll(dynamicConfig)
274+
.build();
275+
257276
restCatalog = initCatalog(currentCatalogName, ImmutableMap.of());
258277
}
259278

@@ -581,7 +600,13 @@ public void testLoadTableWithAccessDelegationForExternalCatalogWithConfigDisable
581600
restCatalog.createNamespace(ns1);
582601
TableMetadata tableMetadata =
583602
TableMetadata.newTableMetadata(
584-
new Schema(List.of(Types.NestedField.of(1, false, "col1", new Types.StringType()))),
603+
new Schema(
604+
List.of(
605+
Types.NestedField.builder()
606+
.withId(1)
607+
.withName("col1")
608+
.ofType(Types.StringType.get())
609+
.build())),
585610
PartitionSpec.unpartitioned(),
586611
externalCatalogBase + "/ns1/my_table",
587612
Map.of());
@@ -616,7 +641,13 @@ public void testLoadTableWithoutAccessDelegationForExternalCatalogWithConfigDisa
616641
restCatalog.createNamespace(ns1);
617642
TableMetadata tableMetadata =
618643
TableMetadata.newTableMetadata(
619-
new Schema(List.of(Types.NestedField.of(1, false, "col1", new Types.StringType()))),
644+
new Schema(
645+
List.of(
646+
Types.NestedField.builder()
647+
.withId(1)
648+
.withName("col1")
649+
.ofType(Types.StringType.get())
650+
.build())),
620651
PartitionSpec.unpartitioned(),
621652
externalCatalogBase + "/ns1/my_table",
622653
Map.of());
@@ -650,7 +681,13 @@ public void testLoadTableWithAccessDelegationForExternalCatalogWithConfigEnabled
650681
restCatalog.createNamespace(ns1);
651682
TableMetadata tableMetadata =
652683
TableMetadata.newTableMetadata(
653-
new Schema(List.of(Types.NestedField.of(1, false, "col1", new Types.StringType()))),
684+
new Schema(
685+
List.of(
686+
Types.NestedField.builder()
687+
.withId(1)
688+
.withName("col1")
689+
.ofType(Types.StringType.get())
690+
.build())),
654691
PartitionSpec.unpartitioned(),
655692
externalCatalogBase + "/ns1/my_table",
656693
Map.of());
@@ -681,7 +718,13 @@ public void testLoadTableTwiceWithETag() {
681718
restCatalog.createNamespace(ns1);
682719
TableMetadata tableMetadata =
683720
TableMetadata.newTableMetadata(
684-
new Schema(List.of(Types.NestedField.of(1, false, "col1", new Types.StringType()))),
721+
new Schema(
722+
List.of(
723+
Types.NestedField.builder()
724+
.withId(1)
725+
.withName("col1")
726+
.ofType(Types.StringType.get())
727+
.build())),
685728
PartitionSpec.unpartitioned(),
686729
"file:///tmp/ns1/my_table",
687730
Map.of());
@@ -728,7 +771,13 @@ public void testRegisterAndLoadTableWithReturnedETag() {
728771
restCatalog.createNamespace(ns1);
729772
TableMetadata tableMetadata =
730773
TableMetadata.newTableMetadata(
731-
new Schema(List.of(Types.NestedField.of(1, false, "col1", new Types.StringType()))),
774+
new Schema(
775+
List.of(
776+
Types.NestedField.builder()
777+
.withId(1)
778+
.withName("col1")
779+
.ofType(Types.StringType.get())
780+
.build())),
732781
PartitionSpec.unpartitioned(),
733782
"file:///tmp/ns1/my_table",
734783
Map.of());
@@ -774,7 +823,13 @@ public void testCreateAndLoadTableWithReturnedEtag() {
774823
restCatalog.createNamespace(ns1);
775824
TableMetadata tableMetadata =
776825
TableMetadata.newTableMetadata(
777-
new Schema(List.of(Types.NestedField.of(1, false, "col1", new Types.StringType()))),
826+
new Schema(
827+
List.of(
828+
Types.NestedField.builder()
829+
.withId(1)
830+
.withName("col1")
831+
.ofType(Types.StringType.get())
832+
.build())),
778833
PartitionSpec.unpartitioned(),
779834
"file:///tmp/ns1/my_table",
780835
Map.of());

‎integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,16 @@ public abstract class PolarisRestCatalogViewIntegrationBase extends ViewCatalogT
6262
Assumptions.setPreferredAssumptionException(PreferredAssumptionException.JUNIT5);
6363
}
6464

65+
public static Map<String, String> DEFAULT_REST_CATALOG_CONFIG =
66+
Map.of(
67+
org.apache.iceberg.CatalogProperties.VIEW_DEFAULT_PREFIX + "key1", "catalog-default-key1",
68+
org.apache.iceberg.CatalogProperties.VIEW_DEFAULT_PREFIX + "key2", "catalog-default-key2",
69+
org.apache.iceberg.CatalogProperties.VIEW_DEFAULT_PREFIX + "key3", "catalog-default-key3",
70+
org.apache.iceberg.CatalogProperties.VIEW_OVERRIDE_PREFIX + "key3",
71+
"catalog-override-key3",
72+
org.apache.iceberg.CatalogProperties.VIEW_OVERRIDE_PREFIX + "key4",
73+
"catalog-override-key4");
74+
6575
private static ClientCredentials adminCredentials;
6676
private static PolarisApiEndpoints endpoints;
6777
private static PolarisClient client;
@@ -120,15 +130,7 @@ public void before(TestInfo testInfo) {
120130

121131
restCatalog =
122132
IcebergHelper.restCatalog(
123-
client,
124-
endpoints,
125-
principalCredentials,
126-
catalogName,
127-
Map.of(
128-
org.apache.iceberg.CatalogProperties.VIEW_DEFAULT_PREFIX + "key1",
129-
"catalog-default-key1",
130-
org.apache.iceberg.CatalogProperties.VIEW_DEFAULT_PREFIX + "key2",
131-
"catalog-default-key2"));
133+
client, endpoints, principalCredentials, catalogName, DEFAULT_REST_CATALOG_CONFIG);
132134
}
133135

134136
@AfterEach

‎plugins/spark/v3.5/integration/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogBaseIT.java‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import java.util.Arrays;
2828
import java.util.Map;
2929
import org.apache.iceberg.exceptions.BadRequestException;
30+
import org.apache.iceberg.exceptions.NamespaceNotEmptyException;
3031
import org.apache.iceberg.spark.SupportsReplaceView;
3132
import org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException;
3233
import org.apache.spark.sql.catalyst.analysis.NoSuchViewException;
@@ -107,7 +108,10 @@ void testNamespaceOperations() throws Exception {
107108

108109
// directly drop lv1ns[0] should fail
109110
assertThatThrownBy(() -> namespaceCatalog.dropNamespace(lv1ns[0], true))
110-
.isInstanceOf(BadRequestException.class);
111+
.isInstanceOfAny(
112+
BadRequestException.class, // Iceberg < 1.9.0
113+
NamespaceNotEmptyException.class // Iceberg >= 1.9.0
114+
);
111115
for (String[] namespace : lv2ns1) {
112116
namespaceCatalog.dropNamespace(namespace, true);
113117
}
@@ -249,7 +253,10 @@ void testListViews() throws Exception {
249253

250254
// drop namespace fails since there are views under it
251255
assertThatThrownBy(() -> namespaceCatalog.dropNamespace(l2ns, true))
252-
.isInstanceOf(BadRequestException.class);
256+
.isInstanceOfAny(
257+
BadRequestException.class, // Iceberg < 1.9.0
258+
NamespaceNotEmptyException.class // Iceberg >= 1.9.0
259+
);
253260
// drop the views
254261
for (String name : nsl2ViewNames) {
255262
viewCatalog.dropView(Identifier.of(l2ns, name));

‎quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -180,6 +180,19 @@ public Map<String, String> getConfigOverrides() {
180180
public static final String SECRET_ACCESS_KEY = "secret_access_key";
181181
public static final String SESSION_TOKEN = "session_token";
182182

183+
public static Map<String, String> TABLE_PREFIXES =
184+
Map.of(
185+
CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key1",
186+
"catalog-default-key1",
187+
CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key2",
188+
"catalog-default-key2",
189+
CatalogProperties.TABLE_DEFAULT_PREFIX + "override-key3",
190+
"catalog-default-key3",
191+
CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key3",
192+
"catalog-override-key3",
193+
CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key4",
194+
"catalog-override-key4");
195+
183196
@Inject MetaStoreManagerFactory managerFactory;
184197
@Inject PolarisConfigurationStore configurationStore;
185198
@Inject PolarisStorageIntegrationProvider storageIntegrationProvider;
@@ -344,6 +357,7 @@ protected IcebergCatalog initCatalog(
344357
ImmutableMap.Builder<String, String> propertiesBuilder =
345358
ImmutableMap.<String, String>builder()
346359
.put(CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO")
360+
.putAll(TABLE_PREFIXES)
347361
.putAll(additionalProperties);
348362
icebergCatalog.initialize(CATALOG_NAME, propertiesBuilder.buildKeepingLast());
349363
return icebergCatalog;

‎quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java‎

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,14 @@ public Map<String, String> getConfigOverrides() {
104104

105105
public static final String CATALOG_NAME = "polaris-catalog";
106106

107+
public static Map<String, String> VIEW_PREFIXES =
108+
Map.of(
109+
CatalogProperties.VIEW_DEFAULT_PREFIX + "key1", "catalog-default-key1",
110+
CatalogProperties.VIEW_DEFAULT_PREFIX + "key2", "catalog-default-key2",
111+
CatalogProperties.VIEW_DEFAULT_PREFIX + "key3", "catalog-default-key3",
112+
CatalogProperties.VIEW_OVERRIDE_PREFIX + "key3", "catalog-override-key3",
113+
CatalogProperties.VIEW_OVERRIDE_PREFIX + "key4", "catalog-override-key4");
114+
107115
@Inject MetaStoreManagerFactory managerFactory;
108116
@Inject UserSecretsManagerFactory userSecretsManagerFactory;
109117
@Inject PolarisConfigurationStore configurationStore;
@@ -211,15 +219,12 @@ public void before(TestInfo testInfo) {
211219
securityContext,
212220
Mockito.mock(),
213221
fileIOFactory);
214-
this.catalog.initialize(
215-
CATALOG_NAME,
216-
ImmutableMap.of(
217-
CatalogProperties.FILE_IO_IMPL,
218-
"org.apache.iceberg.inmemory.InMemoryFileIO",
219-
CatalogProperties.VIEW_DEFAULT_PREFIX + "key1",
220-
"catalog-default-key1",
221-
CatalogProperties.VIEW_DEFAULT_PREFIX + "key2",
222-
"catalog-default-key2"));
222+
Map<String, String> properties =
223+
ImmutableMap.<String, String>builder()
224+
.put(CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO")
225+
.putAll(VIEW_PREFIXES)
226+
.build();
227+
this.catalog.initialize(CATALOG_NAME, properties);
223228
}
224229

225230
@AfterEach

‎quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TaskTestUtils.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,13 @@ static TableMetadata writeTableMetadata(
100100
tmBuilder
101101
.setLocation("path/to/table")
102102
.addSchema(
103-
new Schema(List.of(Types.NestedField.of(1, false, "field1", Types.StringType.get()))))
103+
new Schema(
104+
List.of(
105+
Types.NestedField.builder()
106+
.withId(1)
107+
.withName("field1")
108+
.ofType(Types.StringType.get())
109+
.build())))
104110
.addSortOrder(SortOrder.unsorted())
105111
.assignUUID(UUID.randomUUID().toString())
106112
.addPartitionSpec(PartitionSpec.unpartitioned());

0 commit comments

Comments
 (0)