From 6146a14eb3204fc595ace79bdcbe522b1967b788 Mon Sep 17 00:00:00 2001 From: owenowenisme Date: Mon, 5 May 2025 00:14:08 +0800 Subject: [PATCH 1/7] added Signed-off-by: owenowenisme --- getting-started/eclipselink/docker-compose.yml | 4 ++-- .../in-dev/unreleased/getting-started/using-polaris.md | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/getting-started/eclipselink/docker-compose.yml b/getting-started/eclipselink/docker-compose.yml index 0c2ca17a8b..e10c170042 100644 --- a/getting-started/eclipselink/docker-compose.yml +++ b/getting-started/eclipselink/docker-compose.yml @@ -79,7 +79,7 @@ services: --conf, "spark.sql.catalog.quickstart_catalog.type=rest", --conf, "spark.sql.catalog.quickstart_catalog.warehouse=quickstart_catalog", --conf, "spark.sql.catalog.quickstart_catalog.uri=http://polaris:8181/api/catalog", - --conf, "spark.sql.catalog.quickstart_catalog.credential=root:s3cr3t", + --conf, "spark.sql.catalog.quickstart_catalog.credential=${USER_CLIENT_ID}:${USER_CLIENT_SECRET}", --conf, "spark.sql.catalog.quickstart_catalog.scope=PRINCIPAL_ROLE:ALL", --conf, "spark.sql.defaultCatalog=quickstart_catalog", --conf, "spark.sql.catalogImplementation=in-memory", @@ -96,4 +96,4 @@ services: ports: - "8080:8080" volumes: - - ../assets/trino-config/catalog:/etc/trino/catalog + - ../assets/trino-config/catalog:/etc/trino/catalog \ No newline at end of file diff --git a/site/content/in-dev/unreleased/getting-started/using-polaris.md b/site/content/in-dev/unreleased/getting-started/using-polaris.md index ff83e8748d..46974bcae1 100644 --- a/site/content/in-dev/unreleased/getting-started/using-polaris.md +++ b/site/content/in-dev/unreleased/getting-started/using-polaris.md @@ -174,7 +174,6 @@ Replace the credentials used in the Docker container using the following code: ```shell USER_CLIENT_ID="XXXX" USER_CLIENT_SECRET="YYYY" -sed -i "s/^\(.*spark\.sql\.catalog\.quickstart_catalog\.credential=\).*/\1${USER_CLIENT_ID}:${USER_CLIENT_SECRET}\",/" getting-started/eclipselink/docker-compose.yml docker compose -f getting-started/eclipselink/docker-compose.yml up -d ``` From d2547da5de0bb687e8109b67dc997bb6cd88629a Mon Sep 17 00:00:00 2001 From: owenowenisme Date: Mon, 5 May 2025 00:23:05 +0800 Subject: [PATCH 2/7] fix Signed-off-by: owenowenisme --- getting-started/eclipselink/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/getting-started/eclipselink/docker-compose.yml b/getting-started/eclipselink/docker-compose.yml index e10c170042..39a9dff7b2 100644 --- a/getting-started/eclipselink/docker-compose.yml +++ b/getting-started/eclipselink/docker-compose.yml @@ -96,4 +96,4 @@ services: ports: - "8080:8080" volumes: - - ../assets/trino-config/catalog:/etc/trino/catalog \ No newline at end of file + - ../assets/trino-config/catalog:/etc/trino/catalog From bfafe26a842d09100bb53d4c88d0a2575a35a3ef Mon Sep 17 00:00:00 2001 From: owenowenisme Date: Mon, 5 May 2025 00:52:15 +0800 Subject: [PATCH 3/7] add export Signed-off-by: owenowenisme --- .../in-dev/unreleased/getting-started/using-polaris.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/site/content/in-dev/unreleased/getting-started/using-polaris.md b/site/content/in-dev/unreleased/getting-started/using-polaris.md index 46974bcae1..b0663e16ae 100644 --- a/site/content/in-dev/unreleased/getting-started/using-polaris.md +++ b/site/content/in-dev/unreleased/getting-started/using-polaris.md @@ -172,8 +172,8 @@ Finally, note that we include the `hadoop-aws` package here. If your table is us Replace the credentials used in the Docker container using the following code: ```shell -USER_CLIENT_ID="XXXX" -USER_CLIENT_SECRET="YYYY" +export USER_CLIENT_ID="XXXX" +export USER_CLIENT_SECRET="YYYY" docker compose -f getting-started/eclipselink/docker-compose.yml up -d ``` From d0ad143ae8e45dbc758d529120f76fc563813469 Mon Sep 17 00:00:00 2001 From: owenowenisme Date: Tue, 6 May 2025 15:30:25 +0800 Subject: [PATCH 4/7] update docs using .env Signed-off-by: owenowenisme --- .../assets/polaris/create-catalog.sh | 2 +- .../trino-config/catalog/iceberg.properties | 2 +- .../eclipselink/docker-compose.yml | 4 +++ .../getting-started/using-polaris.md | 32 +++++++++++-------- 4 files changed, 25 insertions(+), 15 deletions(-) diff --git a/getting-started/assets/polaris/create-catalog.sh b/getting-started/assets/polaris/create-catalog.sh index a4e8287690..c35a07cc5a 100755 --- a/getting-started/assets/polaris/create-catalog.sh +++ b/getting-started/assets/polaris/create-catalog.sh @@ -22,7 +22,7 @@ set -e apk add --no-cache jq token=$(curl -s http://polaris:8181/api/catalog/v1/oauth/tokens \ - --user root:s3cr3t \ + --user ${CLIENT_ID}:${CLIENT_SECRET} \ -d grant_type=client_credentials \ -d scope=PRINCIPAL_ROLE:ALL | sed -n 's/.*"access_token":"\([^"]*\)".*/\1/p') diff --git a/getting-started/assets/trino-config/catalog/iceberg.properties b/getting-started/assets/trino-config/catalog/iceberg.properties index 28c3c61faa..176a623b77 100644 --- a/getting-started/assets/trino-config/catalog/iceberg.properties +++ b/getting-started/assets/trino-config/catalog/iceberg.properties @@ -21,7 +21,7 @@ connector.name=iceberg iceberg.catalog.type=rest iceberg.rest-catalog.uri=http://polaris:8181/api/catalog iceberg.rest-catalog.security=OAUTH2 -iceberg.rest-catalog.oauth2.credential=root:s3cr3t +iceberg.rest-catalog.oauth2.credential=${ENV:CLIENT_ID}:${ENV:CLIENT_SECRET} iceberg.rest-catalog.oauth2.scope=PRINCIPAL_ROLE:ALL iceberg.rest-catalog.warehouse=quickstart_catalog # Required to support local filesystem: https://trino.io/docs/current/object-storage.html#configuration diff --git a/getting-started/eclipselink/docker-compose.yml b/getting-started/eclipselink/docker-compose.yml index 39a9dff7b2..866df3eab7 100644 --- a/getting-started/eclipselink/docker-compose.yml +++ b/getting-started/eclipselink/docker-compose.yml @@ -36,6 +36,7 @@ services: polaris.persistence.eclipselink.configuration-file: /deployments/config/eclipselink/persistence.xml polaris.realm-context.realms: POLARIS quarkus.otel.sdk.disabled: "true" + POLARIS_BOOTSTRAP_CREDENTIALS: POLARIS,${CLIENT_ID},${CLIENT_SECRET} volumes: - ../assets/eclipselink/:/deployments/config/eclipselink healthcheck: @@ -54,6 +55,8 @@ services: - STORAGE_LOCATION=${STORAGE_LOCATION} - AWS_ROLE_ARN=${AWS_ROLE_ARN} - AZURE_TENANT_ID=${AZURE_TENANT_ID} + - CLIENT_ID=${CLIENT_ID} + - CLIENT_SECRET=${CLIENT_SECRET} volumes: - ../assets/polaris/:/polaris entrypoint: '/bin/sh -c "chmod +x /polaris/create-catalog.sh && /polaris/create-catalog.sh"' @@ -91,6 +94,7 @@ services: depends_on: polaris-setup: condition: service_completed_successfully + env_file: ../../.env stdin_open: true tty: true ports: diff --git a/site/content/in-dev/unreleased/getting-started/using-polaris.md b/site/content/in-dev/unreleased/getting-started/using-polaris.md index b0663e16ae..b8f522a617 100644 --- a/site/content/in-dev/unreleased/getting-started/using-polaris.md +++ b/site/content/in-dev/unreleased/getting-started/using-polaris.md @@ -21,7 +21,12 @@ Title: Using Polaris type: docs weight: 400 --- - +## Before start +Put your `CLIENT_ID` & `CLIENT_SECRET` in .env file for later use. +```env +CLIENT_ID=YOUR_CLIENT_ID +CLIENT_SECRET=YOUR_CLIENT_SECRET +``` ## Defining a Catalog In Polaris, the [catalog]({{% relref "../entities#catalog" %}}) is the top-level entity that objects like [tables]({{% relref "../entities#table" %}}) and [views]({{% relref "../entities#view" %}}) are organized under. With a Polaris service running, you can create a catalog like so: @@ -79,11 +84,13 @@ With a catalog created, we can create a [principal]({{% relref "../entities#prin Be sure to provide the necessary credentials, hostname, and port as before. -When the `principals create` command completes successfully, it will return the credentials for this new principal. Be sure to note these down for later. For example: +When the `principals create` command completes successfully, it will return the credentials for this new principal. Add the credentials in .env for later. For example: ``` ./polaris ... principals create example {"clientId": "XXXX", "clientSecret": "YYYY"} +echo "USER_CLIENT_ID="XXXX" +USER_CLIENT_SECRET="YYYY"" >> .env ``` Now, we grant the principal the [principal role]({{% relref "../entities#principal-role" %}}) we created, and grant the [catalog role]({{% relref "../entities#catalog-role" %}}) the principal role we created. For more information on these entities, please refer to the linked documentation. @@ -146,6 +153,10 @@ This guide uses [Apache Spark 3.5](https://spark.apache.org/releases/spark-relea _Note: the credentials provided here are those for our principal, not the root credentials._ ```shell +cd polaris +set -a +source .env +cd spark bin/spark-sql \ --packages org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.7.1,org.apache.hadoop:hadoop-aws:3.4.0 \ --conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions \ @@ -154,26 +165,22 @@ bin/spark-sql \ --conf spark.sql.catalog.quickstart_catalog=org.apache.iceberg.spark.SparkCatalog \ --conf spark.sql.catalog.quickstart_catalog.catalog-impl=org.apache.iceberg.rest.RESTCatalog \ --conf spark.sql.catalog.quickstart_catalog.uri=http://localhost:8181/api/catalog \ ---conf spark.sql.catalog.quickstart_catalog.credential='XXXX:YYYY' \ +--conf spark.sql.catalog.quickstart_catalog.credential='${USER_CLIENT_ID}:${USER_CLIENT_SECRET}' \ --conf spark.sql.catalog.quickstart_catalog.scope='PRINCIPAL_ROLE:ALL' \ --conf spark.sql.catalog.quickstart_catalog.token-refresh-enabled=true \ --conf spark.sql.catalog.quickstart_catalog.client.region=us-west-2 ``` -Replace `XXXX` and `YYYY` with the client ID and client secret generated when you created the `quickstart_user` principal. - Similar to the CLI commands above, this configures Spark to use the Polaris running at `localhost:8181`. If your Polaris server is running elsewhere, but sure to update the configuration appropriately. Finally, note that we include the `hadoop-aws` package here. If your table is using a different filesystem, be sure to include the appropriate dependency. #### Using Spark SQL from a Docker container - -Replace the credentials used in the Docker container using the following code: - +Source the .env file before using docker compose. ```shell -export USER_CLIENT_ID="XXXX" -export USER_CLIENT_SECRET="YYYY" +set -a +source .env docker compose -f getting-started/eclipselink/docker-compose.yml up -d ``` @@ -235,9 +242,8 @@ org.apache.iceberg.exceptions.ForbiddenException: Forbidden: Principal 'quicksta Replace the credentials used in the Docker container using the following code: ```shell -USER_CLIENT_ID="XXXX" -USER_CLIENT_SECRET="YYYY" -sed -i "s/^\(iceberg\.rest-catalog\.oauth2\.credential=\).*/\1${USER_CLIENT_ID}:${USER_CLIENT_SECRET}/" getting-started/eclipselink/trino-config/catalog/iceberg.properties +set -a +source .env docker compose -f getting-started/eclipselink/docker-compose.yml down trino docker compose -f getting-started/eclipselink/docker-compose.yml up -d ``` From 0319fc2e5f075a4682f91e3cfb3e921ebc18880e Mon Sep 17 00:00:00 2001 From: owenowenisme Date: Wed, 7 May 2025 01:28:53 +0800 Subject: [PATCH 5/7] update docs Signed-off-by: owenowenisme --- .../unreleased/getting-started/using-polaris.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/site/content/in-dev/unreleased/getting-started/using-polaris.md b/site/content/in-dev/unreleased/getting-started/using-polaris.md index b8f522a617..145147a447 100644 --- a/site/content/in-dev/unreleased/getting-started/using-polaris.md +++ b/site/content/in-dev/unreleased/getting-started/using-polaris.md @@ -21,11 +21,11 @@ Title: Using Polaris type: docs weight: 400 --- -## Before start -Put your `CLIENT_ID` & `CLIENT_SECRET` in .env file for later use. -```env -CLIENT_ID=YOUR_CLIENT_ID -CLIENT_SECRET=YOUR_CLIENT_SECRET +## Setup +Define your `CLIENT_ID` & `CLIENT_SECRET` and put them in .env file for later use. +```shell +echo "CLIENT_ID=YOUR_CLIENT_ID" >> .env +echo "CLIENT_SECRET=YOUR_CLIENT_SECRET" >> .env ``` ## Defining a Catalog @@ -89,8 +89,8 @@ When the `principals create` command completes successfully, it will return the ``` ./polaris ... principals create example {"clientId": "XXXX", "clientSecret": "YYYY"} -echo "USER_CLIENT_ID="XXXX" -USER_CLIENT_SECRET="YYYY"" >> .env +echo "USER_CLIENT_ID=XXXX" >> .env +echo "USER_CLIENT_SECRET=YYYY" >> .env ``` Now, we grant the principal the [principal role]({{% relref "../entities#principal-role" %}}) we created, and grant the [catalog role]({{% relref "../entities#catalog-role" %}}) the principal role we created. For more information on these entities, please refer to the linked documentation. From bd207c881b45278ed7e476181cdd35e5ae824e33 Mon Sep 17 00:00:00 2001 From: owenowenisme Date: Thu, 8 May 2025 22:09:43 +0800 Subject: [PATCH 6/7] change back from using .env to export Signed-off-by: owenowenisme --- .../eclipselink/docker-compose.yml | 4 +++- .../getting-started/using-polaris.md | 23 ++++++------------- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/getting-started/eclipselink/docker-compose.yml b/getting-started/eclipselink/docker-compose.yml index 866df3eab7..516ccf05bd 100644 --- a/getting-started/eclipselink/docker-compose.yml +++ b/getting-started/eclipselink/docker-compose.yml @@ -94,7 +94,9 @@ services: depends_on: polaris-setup: condition: service_completed_successfully - env_file: ../../.env + environment: + - CLIENT_ID=${CLIENT_ID} + - CLIENT_SECRET=${CLIENT_SECRET} stdin_open: true tty: true ports: diff --git a/site/content/in-dev/unreleased/getting-started/using-polaris.md b/site/content/in-dev/unreleased/getting-started/using-polaris.md index 145147a447..f1b80527b7 100644 --- a/site/content/in-dev/unreleased/getting-started/using-polaris.md +++ b/site/content/in-dev/unreleased/getting-started/using-polaris.md @@ -22,10 +22,10 @@ type: docs weight: 400 --- ## Setup -Define your `CLIENT_ID` & `CLIENT_SECRET` and put them in .env file for later use. +Define your `CLIENT_ID` & `CLIENT_SECRET` and export them for future use. ```shell -echo "CLIENT_ID=YOUR_CLIENT_ID" >> .env -echo "CLIENT_SECRET=YOUR_CLIENT_SECRET" >> .env +export CLIENT_ID=YOUR_CLIENT_ID +export CLIENT_SECRET=YOUR_CLIENT_SECRET ``` ## Defining a Catalog @@ -84,13 +84,13 @@ With a catalog created, we can create a [principal]({{% relref "../entities#prin Be sure to provide the necessary credentials, hostname, and port as before. -When the `principals create` command completes successfully, it will return the credentials for this new principal. Add the credentials in .env for later. For example: +When the `principals create` command completes successfully, it will return the credentials for this new principal. Export them for future use. For example: -``` +```shell ./polaris ... principals create example {"clientId": "XXXX", "clientSecret": "YYYY"} -echo "USER_CLIENT_ID=XXXX" >> .env -echo "USER_CLIENT_SECRET=YYYY" >> .env +export USER_CLIENT_ID=XXXX +export USER_CLIENT_SECRET=YYYY ``` Now, we grant the principal the [principal role]({{% relref "../entities#principal-role" %}}) we created, and grant the [catalog role]({{% relref "../entities#catalog-role" %}}) the principal role we created. For more information on these entities, please refer to the linked documentation. @@ -153,10 +153,6 @@ This guide uses [Apache Spark 3.5](https://spark.apache.org/releases/spark-relea _Note: the credentials provided here are those for our principal, not the root credentials._ ```shell -cd polaris -set -a -source .env -cd spark bin/spark-sql \ --packages org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.7.1,org.apache.hadoop:hadoop-aws:3.4.0 \ --conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions \ @@ -177,10 +173,7 @@ Similar to the CLI commands above, this configures Spark to use the Polaris runn Finally, note that we include the `hadoop-aws` package here. If your table is using a different filesystem, be sure to include the appropriate dependency. #### Using Spark SQL from a Docker container -Source the .env file before using docker compose. ```shell -set -a -source .env docker compose -f getting-started/eclipselink/docker-compose.yml up -d ``` @@ -242,8 +235,6 @@ org.apache.iceberg.exceptions.ForbiddenException: Forbidden: Principal 'quicksta Replace the credentials used in the Docker container using the following code: ```shell -set -a -source .env docker compose -f getting-started/eclipselink/docker-compose.yml down trino docker compose -f getting-started/eclipselink/docker-compose.yml up -d ``` From 1e422ded1d0581213d3613640520a92fe123a6c3 Mon Sep 17 00:00:00 2001 From: "Owen Lin (You-Cheng Lin)" <106612301+owenowenisme@users.noreply.github.com> Date: Fri, 9 May 2025 15:37:30 +0800 Subject: [PATCH 7/7] Apply suggestions from code review Co-authored-by: Adnan Hemani --- .../in-dev/unreleased/getting-started/using-polaris.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/site/content/in-dev/unreleased/getting-started/using-polaris.md b/site/content/in-dev/unreleased/getting-started/using-polaris.md index f1b80527b7..5902c9484a 100644 --- a/site/content/in-dev/unreleased/getting-started/using-polaris.md +++ b/site/content/in-dev/unreleased/getting-started/using-polaris.md @@ -173,6 +173,8 @@ Similar to the CLI commands above, this configures Spark to use the Polaris runn Finally, note that we include the `hadoop-aws` package here. If your table is using a different filesystem, be sure to include the appropriate dependency. #### Using Spark SQL from a Docker container + +Refresh the Docker container with the user's credentials: ```shell docker compose -f getting-started/eclipselink/docker-compose.yml up -d ``` @@ -232,7 +234,7 @@ org.apache.iceberg.exceptions.ForbiddenException: Forbidden: Principal 'quicksta ### Connecting with Trino -Replace the credentials used in the Docker container using the following code: +Refresh the Docker container with the user's credentials: ```shell docker compose -f getting-started/eclipselink/docker-compose.yml down trino