From 7b6081ac8691e308883781b7b9f174bd82fd1f74 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Wed, 31 Jul 2024 12:16:07 -0700 Subject: [PATCH 01/14] setting up trino docker, first pass --- docker-compose-trino.yml | 10 ++++++++++ trino-config/catalog/iceberg.properties | 6 ++++++ trino-config/config.properties | 5 +++++ trino-config/jvm.config | 16 ++++++++++++++++ trino-config/node.properties | 2 ++ 5 files changed, 39 insertions(+) create mode 100644 docker-compose-trino.yml create mode 100644 trino-config/catalog/iceberg.properties create mode 100644 trino-config/config.properties create mode 100644 trino-config/jvm.config create mode 100644 trino-config/node.properties diff --git a/docker-compose-trino.yml b/docker-compose-trino.yml new file mode 100644 index 0000000000..b464a72aa1 --- /dev/null +++ b/docker-compose-trino.yml @@ -0,0 +1,10 @@ +version: '3' + +services: + trino: + image: trinodb/trino:latest + ports: + - "8080:8080" + volumes: + - ./trino-config:/etc/trino + network_mode: "host" diff --git a/trino-config/catalog/iceberg.properties b/trino-config/catalog/iceberg.properties new file mode 100644 index 0000000000..94dbd0283b --- /dev/null +++ b/trino-config/catalog/iceberg.properties @@ -0,0 +1,6 @@ +connector.name=iceberg +iceberg.catalog.type=rest +iceberg.rest-catalog.uri=http://localhost:8181/api/catalog +iceberg.rest-catalog.security=OAUTH2 +iceberg.rest-catalog.oauth2.token=principal:root;password:e6d4d1a345e900dc9c82d64d64e075ed;realm:default-realm;role:ALL +iceberg.rest-catalog.warehouse=polaris \ No newline at end of file diff --git a/trino-config/config.properties b/trino-config/config.properties new file mode 100644 index 0000000000..a11cba39db --- /dev/null +++ b/trino-config/config.properties @@ -0,0 +1,5 @@ +#single node install config +coordinator=true +node-scheduler.include-coordinator=true +http-server.http.port=8080 +discovery.uri=http://localhost:8080 diff --git a/trino-config/jvm.config b/trino-config/jvm.config new file mode 100644 index 0000000000..39525b202e --- /dev/null +++ b/trino-config/jvm.config @@ -0,0 +1,16 @@ +-server +-agentpath:/usr/lib/trino/bin/libjvmkill.so +-XX:InitialRAMPercentage=80 +-XX:MaxRAMPercentage=80 +-XX:G1HeapRegionSize=32M +-XX:+ExplicitGCInvokesConcurrent +-XX:+HeapDumpOnOutOfMemoryError +-XX:+ExitOnOutOfMemoryError +-XX:-OmitStackTraceInFastThrow +-XX:ReservedCodeCacheSize=256M +-XX:PerMethodRecompilationCutoff=10000 +-XX:PerBytecodeRecompilationCutoff=10000 +-Djdk.attach.allowAttachSelf=true +-Djdk.nio.maxCachedBufferSize=2000000 +# Allow loading dynamic agent used by JOL +-XX:+EnableDynamicAgentLoading \ No newline at end of file diff --git a/trino-config/node.properties b/trino-config/node.properties new file mode 100644 index 0000000000..08f171287c --- /dev/null +++ b/trino-config/node.properties @@ -0,0 +1,2 @@ +node.environment=docker +node.data-dir=/data/trino \ No newline at end of file From 2b6715b1ea7f39d1c1b5c6784fdadaa85a69c988 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Mon, 5 Aug 2024 08:23:30 -0700 Subject: [PATCH 02/14] mkdir --- docker-compose-trino.yml | 10 ---- getting-started/docker-compose-trino.yml | 51 +++++++++++++++++++ .../trino-config}/catalog/iceberg.properties | 0 .../trino-config}/config.properties | 0 .../trino-config}/jvm.config | 0 .../trino-config}/node.properties | 0 6 files changed, 51 insertions(+), 10 deletions(-) delete mode 100644 docker-compose-trino.yml create mode 100644 getting-started/docker-compose-trino.yml rename {trino-config => getting-started/trino-config}/catalog/iceberg.properties (100%) rename {trino-config => getting-started/trino-config}/config.properties (100%) rename {trino-config => getting-started/trino-config}/jvm.config (100%) rename {trino-config => getting-started/trino-config}/node.properties (100%) diff --git a/docker-compose-trino.yml b/docker-compose-trino.yml deleted file mode 100644 index b464a72aa1..0000000000 --- a/docker-compose-trino.yml +++ /dev/null @@ -1,10 +0,0 @@ -version: '3' - -services: - trino: - image: trinodb/trino:latest - ports: - - "8080:8080" - volumes: - - ./trino-config:/etc/trino - network_mode: "host" diff --git a/getting-started/docker-compose-trino.yml b/getting-started/docker-compose-trino.yml new file mode 100644 index 0000000000..94870f0478 --- /dev/null +++ b/getting-started/docker-compose-trino.yml @@ -0,0 +1,51 @@ +# +# Copyright (c) 2024 Snowflake Computing Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +services: + polaris: + build: + context: .. + ports: + - "8181:8181" + - "8182" + environment: + AWS_REGION: us-west-2 + GOOGLE_APPLICATION_CREDENTIALS: $GOOGLE_APPLICATION_CREDENTIALS + AZURE_TENANT_ID: $AZURE_TENANT_ID + AZURE_CLIENT_ID: $AZURE_CLIENT_ID + AZURE_CLIENT_SECRET: $AZURE_CLIENT_SECRET + command: # override the command to specify aws keys as dropwizard config + - java + - -Ddw.awsAccessKey=$AWS_ACCESS_KEY_ID + - -Ddw.awsSecretKey=$AWS_SECRET_ACCESS_KEY + - -jar + - /app/polaris-service-1.0.0-all.jar + - server + - polaris-server.yml + + healthcheck: + test: ["CMD", "curl", "http://localhost:8182/healthcheck"] + interval: 10s + timeout: 10s + retries: 5 + + trino: + image: trinodb/trino:latest + ports: + - "8080:8080" + volumes: + - ./trino-config:/etc/trino + network_mode: "host" diff --git a/trino-config/catalog/iceberg.properties b/getting-started/trino-config/catalog/iceberg.properties similarity index 100% rename from trino-config/catalog/iceberg.properties rename to getting-started/trino-config/catalog/iceberg.properties diff --git a/trino-config/config.properties b/getting-started/trino-config/config.properties similarity index 100% rename from trino-config/config.properties rename to getting-started/trino-config/config.properties diff --git a/trino-config/jvm.config b/getting-started/trino-config/jvm.config similarity index 100% rename from trino-config/jvm.config rename to getting-started/trino-config/jvm.config diff --git a/trino-config/node.properties b/getting-started/trino-config/node.properties similarity index 100% rename from trino-config/node.properties rename to getting-started/trino-config/node.properties From 919b39081f95cf183152892c609c432999c8fe14 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Mon, 5 Aug 2024 09:30:39 -0700 Subject: [PATCH 03/14] use use principal token --- getting-started/docker-compose-trino.yml | 12 +++++- getting-started/post-init-script.sh | 37 +++++++++++++++++++ .../trino-config/catalog/iceberg.properties | 4 +- 3 files changed, 49 insertions(+), 4 deletions(-) create mode 100644 getting-started/post-init-script.sh diff --git a/getting-started/docker-compose-trino.yml b/getting-started/docker-compose-trino.yml index 94870f0478..a6a674c4b7 100644 --- a/getting-started/docker-compose-trino.yml +++ b/getting-started/docker-compose-trino.yml @@ -41,11 +41,19 @@ services: interval: 10s timeout: 10s retries: 5 - + + post-init-script: + image: curlimages/curl + depends_on: + polaris: + condition: service_healthy + volumes: + - ./post-init-script.sh:/post-init-script.sh + command: ["/bin/sh", "/post-init-script.sh"] + trino: image: trinodb/trino:latest ports: - "8080:8080" volumes: - ./trino-config:/etc/trino - network_mode: "host" diff --git a/getting-started/post-init-script.sh b/getting-started/post-init-script.sh new file mode 100644 index 0000000000..d3ca654a02 --- /dev/null +++ b/getting-started/post-init-script.sh @@ -0,0 +1,37 @@ +# +# Copyright (c) 2024 Snowflake Computing Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +PRINCIPAL_TOKEN="principal:root;realm:default-realm" + +# Use local filesystem by default +curl -i -X POST -H "Authorization: Bearer $PRINCIPAL_TOKEN" -H 'Accept: application/json' -H 'Content-Type: application/json' \ + http://polaris:8181/api/management/v1/catalogs \ + -d '{ + "catalog": { + "name": "polaris", + "type": "INTERNAL", + "readOnly": false, + "properties": { + "default-base-location": "file:///tmp/polaris/" + }, + "storageConfigInfo": { + "storageType": "FILE", + "allowedLocations": [ + "file:///tmp" + ] + } + } + }' diff --git a/getting-started/trino-config/catalog/iceberg.properties b/getting-started/trino-config/catalog/iceberg.properties index 94dbd0283b..124005c927 100644 --- a/getting-started/trino-config/catalog/iceberg.properties +++ b/getting-started/trino-config/catalog/iceberg.properties @@ -1,6 +1,6 @@ connector.name=iceberg iceberg.catalog.type=rest -iceberg.rest-catalog.uri=http://localhost:8181/api/catalog +iceberg.rest-catalog.uri=http://polaris:8181/api/catalog iceberg.rest-catalog.security=OAUTH2 -iceberg.rest-catalog.oauth2.token=principal:root;password:e6d4d1a345e900dc9c82d64d64e075ed;realm:default-realm;role:ALL +iceberg.rest-catalog.oauth2.token=principal:root;realm:default-realm iceberg.rest-catalog.warehouse=polaris \ No newline at end of file From 2224c1129bbe1d9fa47b71ba4b2ec5bf10df46fb Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Mon, 5 Aug 2024 09:33:57 -0700 Subject: [PATCH 04/14] rename --- getting-started/{post-init-script.sh => create-catalog.sh} | 0 getting-started/docker-compose-trino.yml | 6 +++--- 2 files changed, 3 insertions(+), 3 deletions(-) rename getting-started/{post-init-script.sh => create-catalog.sh} (100%) diff --git a/getting-started/post-init-script.sh b/getting-started/create-catalog.sh similarity index 100% rename from getting-started/post-init-script.sh rename to getting-started/create-catalog.sh diff --git a/getting-started/docker-compose-trino.yml b/getting-started/docker-compose-trino.yml index a6a674c4b7..8d471d5d43 100644 --- a/getting-started/docker-compose-trino.yml +++ b/getting-started/docker-compose-trino.yml @@ -42,14 +42,14 @@ services: timeout: 10s retries: 5 - post-init-script: + create-catalog: image: curlimages/curl depends_on: polaris: condition: service_healthy volumes: - - ./post-init-script.sh:/post-init-script.sh - command: ["/bin/sh", "/post-init-script.sh"] + - ./create-catalog.sh:/create-catalog.sh + command: ["/bin/sh", "/create-catalog.sh"] trino: image: trinodb/trino:latest From 9a03f8004f66f2b3e2216c80828f600ea32d70fc Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Wed, 7 Aug 2024 13:39:22 -0700 Subject: [PATCH 05/14] add copyright to all new files --- .../trino-config/catalog/iceberg.properties | 15 +++++++++++++++ getting-started/trino-config/config.properties | 15 +++++++++++++++ getting-started/trino-config/jvm.config | 15 +++++++++++++++ getting-started/trino-config/node.properties | 15 +++++++++++++++ 4 files changed, 60 insertions(+) diff --git a/getting-started/trino-config/catalog/iceberg.properties b/getting-started/trino-config/catalog/iceberg.properties index 124005c927..75c8df4bd9 100644 --- a/getting-started/trino-config/catalog/iceberg.properties +++ b/getting-started/trino-config/catalog/iceberg.properties @@ -1,3 +1,18 @@ +# +# Copyright (c) 2024 Snowflake Computing Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# connector.name=iceberg iceberg.catalog.type=rest iceberg.rest-catalog.uri=http://polaris:8181/api/catalog diff --git a/getting-started/trino-config/config.properties b/getting-started/trino-config/config.properties index a11cba39db..cefef47847 100644 --- a/getting-started/trino-config/config.properties +++ b/getting-started/trino-config/config.properties @@ -1,3 +1,18 @@ +# +# Copyright (c) 2024 Snowflake Computing Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# #single node install config coordinator=true node-scheduler.include-coordinator=true diff --git a/getting-started/trino-config/jvm.config b/getting-started/trino-config/jvm.config index 39525b202e..82f9ec163f 100644 --- a/getting-started/trino-config/jvm.config +++ b/getting-started/trino-config/jvm.config @@ -1,3 +1,18 @@ +# +# Copyright (c) 2024 Snowflake Computing Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# -server -agentpath:/usr/lib/trino/bin/libjvmkill.so -XX:InitialRAMPercentage=80 diff --git a/getting-started/trino-config/node.properties b/getting-started/trino-config/node.properties index 08f171287c..069512c075 100644 --- a/getting-started/trino-config/node.properties +++ b/getting-started/trino-config/node.properties @@ -1,2 +1,17 @@ +# +# Copyright (c) 2024 Snowflake Computing Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# node.environment=docker node.data-dir=/data/trino \ No newline at end of file From c741721c80754b639b136d716594f49d39395943 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Fri, 30 Aug 2024 07:55:21 -0700 Subject: [PATCH 06/14] use ASF license --- getting-started/create-catalog.sh | 25 ++++++++++-------- getting-started/docker-compose-trino.yml | 25 ++++++++++-------- .../trino-config/catalog/iceberg.properties | 26 +++++++++++-------- .../trino-config/config.properties | 26 +++++++++++-------- getting-started/trino-config/jvm.config | 26 +++++++++++-------- getting-started/trino-config/node.properties | 26 +++++++++++-------- 6 files changed, 88 insertions(+), 66 deletions(-) diff --git a/getting-started/create-catalog.sh b/getting-started/create-catalog.sh index d3ca654a02..e5eb963319 100644 --- a/getting-started/create-catalog.sh +++ b/getting-started/create-catalog.sh @@ -1,17 +1,20 @@ # -# Copyright (c) 2024 Snowflake Computing Inc. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 # -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # PRINCIPAL_TOKEN="principal:root;realm:default-realm" diff --git a/getting-started/docker-compose-trino.yml b/getting-started/docker-compose-trino.yml index 8d471d5d43..b9e68c6ee8 100644 --- a/getting-started/docker-compose-trino.yml +++ b/getting-started/docker-compose-trino.yml @@ -1,17 +1,20 @@ # -# Copyright (c) 2024 Snowflake Computing Inc. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 # -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # services: diff --git a/getting-started/trino-config/catalog/iceberg.properties b/getting-started/trino-config/catalog/iceberg.properties index 75c8df4bd9..729377d10e 100644 --- a/getting-started/trino-config/catalog/iceberg.properties +++ b/getting-started/trino-config/catalog/iceberg.properties @@ -1,18 +1,22 @@ # -# Copyright (c) 2024 Snowflake Computing Inc. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 # -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # + connector.name=iceberg iceberg.catalog.type=rest iceberg.rest-catalog.uri=http://polaris:8181/api/catalog diff --git a/getting-started/trino-config/config.properties b/getting-started/trino-config/config.properties index cefef47847..c8afb37998 100644 --- a/getting-started/trino-config/config.properties +++ b/getting-started/trino-config/config.properties @@ -1,18 +1,22 @@ # -# Copyright (c) 2024 Snowflake Computing Inc. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 # -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # + #single node install config coordinator=true node-scheduler.include-coordinator=true diff --git a/getting-started/trino-config/jvm.config b/getting-started/trino-config/jvm.config index 82f9ec163f..ed400e52cc 100644 --- a/getting-started/trino-config/jvm.config +++ b/getting-started/trino-config/jvm.config @@ -1,18 +1,22 @@ # -# Copyright (c) 2024 Snowflake Computing Inc. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 # -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # + -server -agentpath:/usr/lib/trino/bin/libjvmkill.so -XX:InitialRAMPercentage=80 diff --git a/getting-started/trino-config/node.properties b/getting-started/trino-config/node.properties index 069512c075..46cd9a0331 100644 --- a/getting-started/trino-config/node.properties +++ b/getting-started/trino-config/node.properties @@ -1,17 +1,21 @@ # -# Copyright (c) 2024 Snowflake Computing Inc. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 # -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # + node.environment=docker node.data-dir=/data/trino \ No newline at end of file From 1d5161ae8b489df5473aeedb7ec499b78380125b Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Fri, 30 Aug 2024 07:58:35 -0700 Subject: [PATCH 07/14] newline --- getting-started/trino-config/catalog/iceberg.properties | 2 +- getting-started/trino-config/jvm.config | 2 +- getting-started/trino-config/node.properties | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/getting-started/trino-config/catalog/iceberg.properties b/getting-started/trino-config/catalog/iceberg.properties index 729377d10e..a5075530f7 100644 --- a/getting-started/trino-config/catalog/iceberg.properties +++ b/getting-started/trino-config/catalog/iceberg.properties @@ -22,4 +22,4 @@ iceberg.catalog.type=rest iceberg.rest-catalog.uri=http://polaris:8181/api/catalog iceberg.rest-catalog.security=OAUTH2 iceberg.rest-catalog.oauth2.token=principal:root;realm:default-realm -iceberg.rest-catalog.warehouse=polaris \ No newline at end of file +iceberg.rest-catalog.warehouse=polaris diff --git a/getting-started/trino-config/jvm.config b/getting-started/trino-config/jvm.config index ed400e52cc..d57f85ba22 100644 --- a/getting-started/trino-config/jvm.config +++ b/getting-started/trino-config/jvm.config @@ -32,4 +32,4 @@ -Djdk.attach.allowAttachSelf=true -Djdk.nio.maxCachedBufferSize=2000000 # Allow loading dynamic agent used by JOL --XX:+EnableDynamicAgentLoading \ No newline at end of file +-XX:+EnableDynamicAgentLoading diff --git a/getting-started/trino-config/node.properties b/getting-started/trino-config/node.properties index 46cd9a0331..3eed9b5f26 100644 --- a/getting-started/trino-config/node.properties +++ b/getting-started/trino-config/node.properties @@ -18,4 +18,4 @@ # node.environment=docker -node.data-dir=/data/trino \ No newline at end of file +node.data-dir=/data/trino From 081cc850b98d547e306e198da993ee3d789f34d1 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Fri, 30 Aug 2024 08:10:07 -0700 Subject: [PATCH 08/14] use trino docker image default config --- getting-started/docker-compose-trino.yml | 2 +- .../trino-config/config.properties | 24 ------------- getting-started/trino-config/jvm.config | 35 ------------------- getting-started/trino-config/node.properties | 21 ----------- 4 files changed, 1 insertion(+), 81 deletions(-) delete mode 100644 getting-started/trino-config/config.properties delete mode 100644 getting-started/trino-config/jvm.config delete mode 100644 getting-started/trino-config/node.properties diff --git a/getting-started/docker-compose-trino.yml b/getting-started/docker-compose-trino.yml index b9e68c6ee8..539e4357fc 100644 --- a/getting-started/docker-compose-trino.yml +++ b/getting-started/docker-compose-trino.yml @@ -59,4 +59,4 @@ services: ports: - "8080:8080" volumes: - - ./trino-config:/etc/trino + - ./trino-config/catalog:/etc/trino/catalog diff --git a/getting-started/trino-config/config.properties b/getting-started/trino-config/config.properties deleted file mode 100644 index c8afb37998..0000000000 --- a/getting-started/trino-config/config.properties +++ /dev/null @@ -1,24 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -#single node install config -coordinator=true -node-scheduler.include-coordinator=true -http-server.http.port=8080 -discovery.uri=http://localhost:8080 diff --git a/getting-started/trino-config/jvm.config b/getting-started/trino-config/jvm.config deleted file mode 100644 index d57f85ba22..0000000000 --- a/getting-started/trino-config/jvm.config +++ /dev/null @@ -1,35 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - --server --agentpath:/usr/lib/trino/bin/libjvmkill.so --XX:InitialRAMPercentage=80 --XX:MaxRAMPercentage=80 --XX:G1HeapRegionSize=32M --XX:+ExplicitGCInvokesConcurrent --XX:+HeapDumpOnOutOfMemoryError --XX:+ExitOnOutOfMemoryError --XX:-OmitStackTraceInFastThrow --XX:ReservedCodeCacheSize=256M --XX:PerMethodRecompilationCutoff=10000 --XX:PerBytecodeRecompilationCutoff=10000 --Djdk.attach.allowAttachSelf=true --Djdk.nio.maxCachedBufferSize=2000000 -# Allow loading dynamic agent used by JOL --XX:+EnableDynamicAgentLoading diff --git a/getting-started/trino-config/node.properties b/getting-started/trino-config/node.properties deleted file mode 100644 index 3eed9b5f26..0000000000 --- a/getting-started/trino-config/node.properties +++ /dev/null @@ -1,21 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -node.environment=docker -node.data-dir=/data/trino From 98fb0833256b5bdf65f2be0dc2b9fa614dac4f5d Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Fri, 30 Aug 2024 08:13:04 -0700 Subject: [PATCH 09/14] move to trino/ subdir --- getting-started/{ => trino}/create-catalog.sh | 0 getting-started/{ => trino}/docker-compose-trino.yml | 0 .../{ => trino}/trino-config/catalog/iceberg.properties | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename getting-started/{ => trino}/create-catalog.sh (100%) rename getting-started/{ => trino}/docker-compose-trino.yml (100%) rename getting-started/{ => trino}/trino-config/catalog/iceberg.properties (100%) diff --git a/getting-started/create-catalog.sh b/getting-started/trino/create-catalog.sh similarity index 100% rename from getting-started/create-catalog.sh rename to getting-started/trino/create-catalog.sh diff --git a/getting-started/docker-compose-trino.yml b/getting-started/trino/docker-compose-trino.yml similarity index 100% rename from getting-started/docker-compose-trino.yml rename to getting-started/trino/docker-compose-trino.yml diff --git a/getting-started/trino-config/catalog/iceberg.properties b/getting-started/trino/trino-config/catalog/iceberg.properties similarity index 100% rename from getting-started/trino-config/catalog/iceberg.properties rename to getting-started/trino/trino-config/catalog/iceberg.properties From 2af27cdbe92540fd35331c2f377878533881aa83 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Fri, 30 Aug 2024 08:15:01 -0700 Subject: [PATCH 10/14] rename to create-polaris-catalog.sh --- .../trino/{create-catalog.sh => create-polaris-catalog.sh} | 0 getting-started/trino/docker-compose-trino.yml | 6 +++--- 2 files changed, 3 insertions(+), 3 deletions(-) rename getting-started/trino/{create-catalog.sh => create-polaris-catalog.sh} (100%) diff --git a/getting-started/trino/create-catalog.sh b/getting-started/trino/create-polaris-catalog.sh similarity index 100% rename from getting-started/trino/create-catalog.sh rename to getting-started/trino/create-polaris-catalog.sh diff --git a/getting-started/trino/docker-compose-trino.yml b/getting-started/trino/docker-compose-trino.yml index 539e4357fc..b00b9d1571 100644 --- a/getting-started/trino/docker-compose-trino.yml +++ b/getting-started/trino/docker-compose-trino.yml @@ -45,14 +45,14 @@ services: timeout: 10s retries: 5 - create-catalog: + create-polaris-catalog: image: curlimages/curl depends_on: polaris: condition: service_healthy volumes: - - ./create-catalog.sh:/create-catalog.sh - command: ["/bin/sh", "/create-catalog.sh"] + - ./create-polaris-catalog.sh:/create-polaris-catalog.sh + command: ["/bin/sh", "/create-polaris-catalog.sh"] trino: image: trinodb/trino:latest From feb24e0e735e58e9ea71d467c3d2331d131ef26b Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Fri, 30 Aug 2024 08:16:28 -0700 Subject: [PATCH 11/14] fix docker build --- getting-started/trino/docker-compose-trino.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/getting-started/trino/docker-compose-trino.yml b/getting-started/trino/docker-compose-trino.yml index b00b9d1571..804f321656 100644 --- a/getting-started/trino/docker-compose-trino.yml +++ b/getting-started/trino/docker-compose-trino.yml @@ -20,7 +20,7 @@ services: polaris: build: - context: .. + context: ../.. ports: - "8181:8181" - "8182" From 9c2be7f6cb270ec47944987de652535f00a5350a Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Fri, 30 Aug 2024 08:28:05 -0700 Subject: [PATCH 12/14] recopy over polaris docker-compose --- getting-started/trino/docker-compose-trino.yml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/getting-started/trino/docker-compose-trino.yml b/getting-started/trino/docker-compose-trino.yml index 804f321656..73cb517da9 100644 --- a/getting-started/trino/docker-compose-trino.yml +++ b/getting-started/trino/docker-compose-trino.yml @@ -20,7 +20,7 @@ services: polaris: build: - context: ../.. + context: . ports: - "8181:8181" - "8182" @@ -30,14 +30,8 @@ services: AZURE_TENANT_ID: $AZURE_TENANT_ID AZURE_CLIENT_ID: $AZURE_CLIENT_ID AZURE_CLIENT_SECRET: $AZURE_CLIENT_SECRET - command: # override the command to specify aws keys as dropwizard config - - java - - -Ddw.awsAccessKey=$AWS_ACCESS_KEY_ID - - -Ddw.awsSecretKey=$AWS_SECRET_ACCESS_KEY - - -jar - - /app/polaris-service-1.0.0-all.jar - - server - - polaris-server.yml + # add aws keys as dropwizard config + JAVA_OPTS: -Ddw.awsAccessKey=$AWS_ACCESS_KEY_ID -Ddw.awsSecretKey=$AWS_SECRET_ACCESS_KEY healthcheck: test: ["CMD", "curl", "http://localhost:8182/healthcheck"] From ca2729720596e41f6ec55788f45dbc40eff72d31 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Fri, 30 Aug 2024 08:46:10 -0700 Subject: [PATCH 13/14] add README --- getting-started/trino/README.md | 52 +++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 getting-started/trino/README.md diff --git a/getting-started/trino/README.md b/getting-started/trino/README.md new file mode 100644 index 0000000000..91031340f9 --- /dev/null +++ b/getting-started/trino/README.md @@ -0,0 +1,52 @@ + + +# Getting Started with Trino and Apache Polaris + +This getting started guide provides a docker-compose file to set up [Trino](https://trino.io/) with Apache Polaris. Apache Polaris is configured as an Iceberg REST Catalog in Trino. + +## Run the docker-compose file +To start the docker-compose file, run this command from the repo's root directory: +``` +docker-compose -f getting-started/trino/docker-compose-trino.yml up +``` + +## Run Trino queries via Trino ClI +To access the Trino CLI, run this command +``` +docker exec -it trino-trino-1 trino +``` +Note, `trino-trino-1` is the name docker container. + +Example Trino queries: +``` +SHOW CATALOGS; +SHOW SCHEMAS FROM iceberg; +SHOW TABLES FROM iceberg.information_schema; +DESCRIBE iceberg.information_schema.tables; + +CREATE SCHEMA iceberg.tpch; +CREATE TABLE iceberg.tpch.test_polaris AS SELECT 1 x; +SELECT * FROM iceberg.tpch.test_polaris; +``` + +## Note +The polaris catalog setup script use the credential `principal:root;realm:default-realm`. This credential is used so users do not need to fetch credentials from Apache Polaris' console output. + +An example catalog is created in Apache Polaris using the `curl` command. See `create-polaris-catalog.sh` for details. From b3e18d644799ce40d590a8a2890eb5385d46e36f Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Thu, 12 Sep 2024 22:26:47 -0700 Subject: [PATCH 14/14] context: ../../ --- getting-started/trino/docker-compose-trino.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/getting-started/trino/docker-compose-trino.yml b/getting-started/trino/docker-compose-trino.yml index 73cb517da9..3c9ab36f5d 100644 --- a/getting-started/trino/docker-compose-trino.yml +++ b/getting-started/trino/docker-compose-trino.yml @@ -20,7 +20,7 @@ services: polaris: build: - context: . + context: ../../ ports: - "8181:8181" - "8182"