Skip to content

Commit da26e4c

Browse files
adutraadutra
authored
Add more getting-started examples with Polaris (#859)
1 parent 6e165ff commit da26e4c

File tree

12 files changed

+542
-12
lines changed

12 files changed

+542
-12
lines changed

‎README.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,9 @@ select * from db1.table1;
8080
build the image locally.
8181
- `docker run -p 8181:8181 -p 8182:8182 apache/polaris:latest` - To run the image.
8282

83+
The Polaris codebase contains some docker compose examples to quickly get started with Polaris,
84+
using different configurations. Check the `./getting-started` directory for more information.
85+
8386
#### Running in Kubernetes
8487

8588
- `./run.sh` - To run Polaris as a mini-deployment locally. This will create a Kind cluster,

‎getting-started/README.md‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
<!--
2+
Licensed to the Apache Software Foundation (ASF) under one
3+
or more contributor license agreements. See the NOTICE file
4+
distributed with this work for additional information
5+
regarding copyright ownership. The ASF licenses this file
6+
to you under the Apache License, Version 2.0 (the
7+
"License"); you may not use this file except in compliance
8+
with the License. You may obtain a copy of the License at
9+
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
12+
Unless required by applicable law or agreed to in writing,
13+
software distributed under the License is distributed on an
14+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
KIND, either express or implied. See the License for the
16+
specific language governing permissions and limitations
17+
under the License.
18+
-->
19+
20+
# Getting Started with Apache Polaris
21+
22+
You can quickly get started with Polaris by playing with the docker-compose examples provided in
23+
this directory. Each example has detailed instructions.
24+
25+
## Prerequisites
26+
27+
- [Docker](https://docs.docker.com/get-docker/)
28+
- [Docker Compose](https://docs.docker.com/compose/install/)
29+
- [jq](https://stedolan.github.io/jq/download/) (for some examples)
30+
31+
## Getting Started Examples
32+
33+
- [Spark](spark): An example that uses an in-memory metastore, automatically bootstrapped, with
34+
Apache Spark and a Jupyter notebook.
35+
36+
- [Trino](trino): An example that uses Trino with Polaris.
37+
38+
- [Telemetry](telemetry): An example that includes Prometheus and Jaeger to collect metrics and
39+
traces from Apache Polaris. This example automatically creates a `polaris_demo` catalog.
40+
41+
- [Eclipselink](elipselink): An example that uses an Eclipselink metastore and a Postgres
42+
database. The realm is bootstrapped with the Polaris Admin tool. This example also creates a
43+
`polaris_demo` catalog, and offers the ability to run Spark SQL queries. Finally, it shows how to
44+
attach a debugger to the Polaris server.

‎getting-started/assets/eclipselink/persistence.xml‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
<!--
2+
~ Licensed to the Apache Software Foundation (ASF) under one
3+
~ or more contributor license agreements. See the NOTICE file
4+
~ distributed with this work for additional information
5+
~ regarding copyright ownership. The ASF licenses this file
6+
~ to you under the Apache License, Version 2.0 (the
7+
~ "License"); you may not use this file except in compliance
8+
~ with the License. You may obtain a copy of the License at
9+
~
10+
~ http://www.apache.org/licenses/LICENSE-2.0
11+
~
12+
~ Unless required by applicable law or agreed to in writing,
13+
~ software distributed under the License is distributed on an
14+
~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
~ KIND, either express or implied. See the License for the
16+
~ specific language governing permissions and limitations
17+
~ under the License.
18+
-->
19+
20+
<persistence version="2.0" xmlns="http://java.sun.com/xml/ns/persistence"
21+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
22+
xsi:schemaLocation="http://java.sun.com/xml/ns/persistence http://java.sun.com/xml/ns/persistence/persistence_2_0.xsd">
23+
24+
<persistence-unit name="polaris" transaction-type="RESOURCE_LOCAL">
25+
<provider>org.eclipse.persistence.jpa.PersistenceProvider</provider>
26+
<class>org.apache.polaris.jpa.models.ModelEntity</class>
27+
<class>org.apache.polaris.jpa.models.ModelEntityActive</class>
28+
<class>org.apache.polaris.jpa.models.ModelEntityChangeTracking</class>
29+
<class>org.apache.polaris.jpa.models.ModelEntityDropped</class>
30+
<class>org.apache.polaris.jpa.models.ModelGrantRecord</class>
31+
<class>org.apache.polaris.jpa.models.ModelPrincipalSecrets</class>
32+
<class>org.apache.polaris.jpa.models.ModelSequenceId</class>
33+
<shared-cache-mode>NONE</shared-cache-mode>
34+
<properties>
35+
<property name="jakarta.persistence.jdbc.url"
36+
value="jdbc:postgresql://postgres:5432/{realm}"/>
37+
<property name="jakarta.persistence.jdbc.user" value="postgres"/>
38+
<property name="jakarta.persistence.jdbc.password" value="postgres"/>
39+
<property name="jakarta.persistence.schema-generation.database.action" value="create"/>
40+
<property name="eclipselink.persistence-context.flush-mode" value="auto"/>
41+
</properties>
42+
</persistence-unit>
43+
</persistence>

‎getting-started/assets/polaris/create-catalog.sh‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#
19+
20+
set -e
21+
22+
token=$(curl -s http://polaris:8181/api/catalog/v1/oauth/tokens \
23+
--user root:s3cr3t \
24+
-d grant_type=client_credentials \
25+
-d scope=PRINCIPAL_ROLE:ALL | sed -n 's/.*"access_token":"\([^"]*\)".*/\1/p')
26+
27+
if [ -z "${token}" ]; then
28+
echo "Failed to obtain access token."
29+
exit 1
30+
fi
31+
32+
echo
33+
echo "Obtained access token: ${token}"
34+
35+
echo
36+
echo Creating a catalog named polaris_demo...
37+
38+
curl -s -H "Authorization: Bearer ${token}" \
39+
-H 'Accept: application/json' \
40+
-H 'Content-Type: application/json' \
41+
http://polaris:8181/api/management/v1/catalogs \
42+
-d '{
43+
"catalog": {
44+
"name": "polaris_demo",
45+
"type": "INTERNAL",
46+
"readOnly": false,
47+
"properties": {
48+
"default-base-location": "file:///tmp/polaris/"
49+
},
50+
"storageConfigInfo": {
51+
"storageType": "FILE",
52+
"allowedLocations": [
53+
"file:///tmp"
54+
]
55+
}
56+
}
57+
}'
58+
59+
echo
60+
echo Done.

‎getting-started/assets/prometheus/prometheus.yml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#
19+
20+
global:
21+
scrape_interval: 15s
22+
evaluation_interval: 15s
23+
scrape_configs:
24+
- job_name: 'polaris'
25+
scrape_interval: 5s
26+
metrics_path: /q/metrics
27+
static_configs:
28+
- targets: ['polaris:8182']
29+
labels:
30+
service: polaris

‎getting-started/elipselink/README.md‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
<!--
2+
Licensed to the Apache Software Foundation (ASF) under one
3+
or more contributor license agreements. See the NOTICE file
4+
distributed with this work for additional information
5+
regarding copyright ownership. The ASF licenses this file
6+
to you under the Apache License, Version 2.0 (the
7+
"License"); you may not use this file except in compliance
8+
with the License. You may obtain a copy of the License at
9+
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
12+
Unless required by applicable law or agreed to in writing,
13+
software distributed under the License is distributed on an
14+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
KIND, either express or implied. See the License for the
16+
specific language governing permissions and limitations
17+
under the License.
18+
-->
19+
20+
# Getting Started with Apache Polaris, EclipseLink, Postgres and Spark SQL
21+
22+
This example requires `jq` to be installed on your machine.
23+
24+
1. If such an image is not already present, build the Polaris image with support for EclipseLink and
25+
the Postgres JDBC driver:
26+
27+
```shell
28+
./gradlew :polaris-quarkus-server:assemble :polaris-quarkus-admin:assemble \
29+
-PeclipseLinkDeps=org.postgresql:postgresql:42.7.4 \
30+
-Dquarkus.container-image.tag=postgres-latest \
31+
-Dquarkus.container-image.build=true
32+
```
33+
34+
2. Start the docker compose group by running the following command from the root of the repository:
35+
36+
```shell
37+
docker compose -f getting-started/eclipselink/docker-compose.yml up
38+
```
39+
40+
3. Using spark-sql: attach to the running spark-sql container:
41+
42+
```shell
43+
docker attach $(docker ps -q --filter name=spark-sql)
44+
```
45+
46+
You may not see Spark's prompt immediately, type ENTER to see it. A few commands that you can try:
47+
48+
```sql
49+
CREATE NAMESPACE polaris.ns1;
50+
USE polaris.ns1;
51+
CREATE TABLE table1 (id int, name string);
52+
INSERT INTO table1 VALUES (1, 'a');
53+
SELECT * FROM table1;
54+
```
55+
56+
4. To access Polaris from the host machine, first request an access token:
57+
58+
```shell
59+
export POLARIS_TOKEN=$(curl -s http://polaris:8181/api/catalog/v1/oauth/tokens \
60+
--resolve polaris:8181:127.0.0.1 \
61+
--user root:s3cr3t \
62+
-d 'grant_type=client_credentials' \
63+
-d 'scope=PRINCIPAL_ROLE:ALL' | jq -r .access_token)
64+
```
65+
66+
5. Then, use the access token in the Authorization header when accessing Polaris:
67+
68+
```shell
69+
curl -v http://127.0.0.1:8181/api/management/v1/principal-roles -H "Authorization: Bearer $POLARIS_TOKEN"
70+
curl -v http://127.0.0.1:8181/api/catalog/v1/config?warehouse=polaris_demo -H "Authorization: Bearer $POLARIS_TOKEN"
71+
```

‎getting-started/elipselink/docker-compose.yml‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#
19+
20+
services:
21+
22+
polaris:
23+
# IMPORTANT: the image MUST contain the Postgres JDBC driver and EclipseLink dependencies, see README for instructions
24+
image: apache/polaris:postgres-latest
25+
ports:
26+
# API port
27+
- "8181:8181"
28+
# Management port (metrics and health checks)
29+
- "8182:8182"
30+
# Optional, allows attaching a debugger to the Polaris JVM
31+
- "5005:5005"
32+
depends_on:
33+
polaris-bootstrap:
34+
condition: service_completed_successfully
35+
postgres:
36+
condition: service_healthy
37+
environment:
38+
JAVA_DEBUG: "true"
39+
JAVA_DEBUG_PORT: "*:5005"
40+
polaris.persistence.type: eclipse-link
41+
polaris.persistence.eclipselink.configuration-file: /deployments/config/eclipselink/persistence.xml
42+
polaris.realm-context.realms: POLARIS
43+
quarkus.otel.sdk.disabled: "true"
44+
volumes:
45+
- ../assets/eclipselink/:/deployments/config/eclipselink
46+
healthcheck:
47+
test: ["CMD", "curl", "http://localhost:8182/q/health"]
48+
interval: 2s
49+
timeout: 10s
50+
retries: 10
51+
52+
polaris-bootstrap:
53+
# IMPORTANT: the image MUST contain the Postgres JDBC driver and EclipseLink dependencies, see README for instructions
54+
image: apache/polaris-admin-tool:postgres-latest
55+
depends_on:
56+
postgres:
57+
condition: service_healthy
58+
environment:
59+
polaris.persistence.type: eclipse-link
60+
polaris.persistence.eclipselink.configuration-file: /deployments/config/eclipselink/persistence.xml
61+
volumes:
62+
- ../assets/eclipselink/:/deployments/config/eclipselink
63+
command:
64+
- "bootstrap"
65+
- "--realm=POLARIS"
66+
- "--credential=POLARIS,root,s3cr3t"
67+
68+
polaris-setup:
69+
image: alpine/curl
70+
depends_on:
71+
polaris:
72+
condition: service_healthy
73+
volumes:
74+
- ../assets/polaris/:/polaris
75+
entrypoint: '/bin/sh -c "chmod +x /polaris/create-catalog.sh && /polaris/create-catalog.sh"'
76+
77+
postgres:
78+
image: postgres:17.2
79+
ports:
80+
- "5432:5432"
81+
# set shared memory limit when using docker-compose
82+
shm_size: 128mb
83+
environment:
84+
POSTGRES_USER: postgres
85+
POSTGRES_PASSWORD: postgres
86+
POSTGRES_DB: POLARIS
87+
POSTGRES_INITDB_ARGS: "--encoding UTF8 --data-checksums"
88+
healthcheck:
89+
test: "pg_isready -U postgres"
90+
interval: 5s
91+
timeout: 2s
92+
retries: 15
93+
94+
spark-sql:
95+
image: apache/spark:3.5.4-java17-python3
96+
depends_on:
97+
polaris-setup:
98+
condition: service_completed_successfully
99+
stdin_open: true
100+
tty: true
101+
ports:
102+
- "4040-4045:4040-4045"
103+
healthcheck:
104+
test: "curl localhost:4040"
105+
interval: 5s
106+
retries: 15
107+
command: [
108+
/opt/spark/bin/spark-sql,
109+
--packages, "org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.7.0,software.amazon.awssdk:bundle:2.28.17,software.amazon.awssdk:url-connection-client:2.28.17",
110+
--conf, "spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions",
111+
--conf, "spark.sql.catalog.polaris=org.apache.iceberg.spark.SparkCatalog",
112+
--conf, "spark.sql.catalog.polaris.type=rest",
113+
--conf, "spark.sql.catalog.polaris.warehouse=polaris_demo",
114+
--conf, "spark.sql.catalog.polaris.uri=http://polaris:8181/api/catalog",
115+
--conf, "spark.sql.catalog.polaris.credential=root:s3cr3t",
116+
--conf, "spark.sql.catalog.polaris.scope=PRINCIPAL_ROLE:ALL",
117+
--conf, "spark.sql.defaultCatalog=polaris",
118+
--conf, "spark.sql.catalogImplementation=in-memory",
119+
]
120+
volumes:
121+
- ~/.ivy2:/home/spark/.ivy2

0 commit comments

Comments
 (0)