From b257100790d3ac9e2669033ae14adee360413039 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 23 Jul 2025 20:27:33 -0400 Subject: [PATCH 1/7] opensearch docker compose --- Makefile | 96 ++++++++++++++++++++++++++++++++ docker-compose.yaml | 63 ++++++++++++++++++--- opensearch/config/opensearch.yml | 19 +++++++ 3 files changed, 171 insertions(+), 7 deletions(-) create mode 100644 Makefile create mode 100644 opensearch/config/opensearch.yml diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..db7e242a --- /dev/null +++ b/Makefile @@ -0,0 +1,96 @@ +# STAC Auth Proxy Makefile +# Easily manage different backend configurations + +.PHONY: help up-pgstac up-opensearch up-all down clean logs ps + +# Default backend +BACKEND ?= pgstac + +help: + @echo "STAC Auth Proxy - Backend Management" + @echo "" + @echo "Available commands:" + @echo " make up-pgstac - Start with PostgreSQL/pgSTAC backend" + @echo " make up-opensearch - Start with OpenSearch backend" + @echo " make up-all - Start both backends (ports will conflict)" + @echo " make down - Stop all services" + @echo " make clean - Stop services and remove volumes" + @echo " make logs - View logs (use BACKEND=pgstac|opensearch)" + @echo " make ps - List running services" + @echo "" + @echo "Environment variables:" + @echo " STAC_PORT - STAC API port (default: 8001)" + @echo " UPSTREAM_URL - Proxy upstream URL (auto-configured)" + +# Start with pgSTAC backend +up-pgstac: + @echo "Starting with pgSTAC backend..." + @export COMPOSE_PROFILES=pgstac && \ + export UPSTREAM_URL=http://stac:8001 && \ + docker-compose up -d + @echo "" + @echo "Services started:" + @echo " STAC API (pgSTAC): http://localhost:8001" + @echo " Auth Proxy: http://localhost:8000" + @echo " Mock OIDC: http://localhost:8888" + @echo " PostgreSQL: localhost:5439" + +# Start with OpenSearch backend +up-opensearch: + @echo "Starting with OpenSearch backend..." + @export COMPOSE_PROFILES=opensearch && \ + export UPSTREAM_URL=http://stac-opensearch:8001 && \ + docker-compose up -d + @echo "" + @echo "Services started:" + @echo " STAC API (OpenSearch): http://localhost:8001" + @echo " Auth Proxy: http://localhost:8000" + @echo " Mock OIDC: http://localhost:8888" + @echo " OpenSearch: http://localhost:9200" + +# Start all services (for development/testing) +up-all: + @echo "Starting all services (both backends)..." + @echo "WARNING: Both STAC APIs will try to use port 8001" + @export COMPOSE_PROFILES=all && \ + docker-compose up -d + +# Stop all services +down: + docker-compose --profile pgstac --profile opensearch down + +# Clean up (stop and remove volumes) +clean: + docker-compose --profile pgstac --profile opensearch down -v + rm -rf .pgdata + +# View logs +logs: + @if [ "$(BACKEND)" = "opensearch" ]; then \ + docker-compose --profile opensearch logs -f stac-os database-os proxy; \ + else \ + docker-compose --profile pgstac logs -f stac-pg database-pg proxy; \ + fi + +# List running services +ps: + docker-compose --profile pgstac --profile opensearch ps + +# Test endpoints +test-stac: + @echo "Testing STAC API..." + @curl -s http://localhost:8001 | jq . + @echo "" + @echo "Testing Auth Proxy..." + @curl -s http://localhost:8000 | jq . + +# Initialize pgSTAC database +init-pgstac: + @echo "Initializing pgSTAC database..." + docker-compose --profile pgstac exec database-pg psql -U username -d postgis -c "CREATE EXTENSION IF NOT EXISTS postgis;" + docker-compose --profile pgstac exec database-pg psql -U username -d postgis -c "CREATE EXTENSION IF NOT EXISTS btree_gist;" + +# Initialize OpenSearch index +init-opensearch: + @echo "Initializing OpenSearch indices..." + @echo "This would typically be done by the STAC API on first run" \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index 78c9d5e8..9abfe2b5 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,5 +1,7 @@ services: - stac: +# PostGRES STAC API + stac-pg: + profiles: ["pg"] image: ghcr.io/stac-utils/stac-fastapi-pgstac:5.0.2 environment: APP_HOST: 0.0.0.0 @@ -22,10 +24,40 @@ services: ports: - "8001:8001" depends_on: - - database - command: bash -c "./scripts/wait-for-it.sh database:5432 && python -m stac_fastapi.pgstac.app" + - database-pg + command: bash -c "./scripts/wait-for-it.sh database-pg:5432 && python -m stac_fastapi.pgstac.app" - database: +# Opensearch STAC API + stac-os: + profiles: ["os"] + container_name: stac-fastapi-os + image: ghcr.io/stac-utils/stac-fastapi-os:latest + hostname: stac-os + environment: + - STAC_FASTAPI_TITLE=stac-fastapi-opensearch + - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Opensearch backend + - STAC_FASTAPI_VERSION=6.0.0 + - STAC_FASTAPI_LANDING_PAGE_ID=stac-fastapi-opensearch + - APP_HOST=0.0.0.0 + - APP_PORT=8001 + - RELOAD=true + - ENVIRONMENT=local + - WEB_CONCURRENCY=10 + - ES_HOST=database-os + - ES_PORT=9202 + - ES_USE_SSL=false + - ES_VERIFY_CERTS=false + - BACKEND=opensearch + - STAC_FASTAPI_RATE_LIMIT=200/minute + ports: + - "8001:8001" + depends_on: + - database-os + command: + bash -c "./scripts/wait-for-it-es.sh database-os:9202 && python -m stac_fastapi.opensearch.app" + + database-pg: + profiles: ["pg"] image: ghcr.io/stac-utils/pgstac:v0.9.5 environment: POSTGRES_USER: username @@ -40,13 +72,29 @@ services: volumes: - ./.pgdata:/var/lib/postgresql/data + database-os: + profiles: ["os"] + container_name: database-os + image: opensearchproject/opensearch:${OPENSEARCH_VERSION:-2.11.1} + hostname: database-os + environment: + - discovery.type=single-node + - plugins.security.disabled=true + - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m + volumes: + - ./opensearch/config/opensearch.yml:/usr/share/opensearch/config/opensearch.yml + - ./opensearch/snapshots:/usr/share/opensearch/snapshots + ports: + - "9202:9202" + proxy: + profiles: ["pg", "os"] depends_on: - - stac + - oidc build: context: . environment: - UPSTREAM_URL: ${UPSTREAM_URL:-http://stac:8001} + UPSTREAM_URL: ${UPSTREAM_URL:-http://stac-pg:8001} OIDC_DISCOVERY_URL: ${OIDC_DISCOVERY_URL:-http://localhost:8888/.well-known/openid-configuration} OIDC_DISCOVERY_INTERNAL_URL: ${OIDC_DISCOVERY_INTERNAL_URL:-http://oidc:8888/.well-known/openid-configuration} env_file: @@ -58,6 +106,7 @@ services: - ./src:/app/src oidc: + profiles: ["pg", "os"] image: ghcr.io/alukach/mock-oidc-server:latest environment: ISSUER: http://localhost:8888 @@ -67,5 +116,5 @@ services: - "8888:8888" networks: - default: + eoapi-network: name: eoapi-network diff --git a/opensearch/config/opensearch.yml b/opensearch/config/opensearch.yml new file mode 100644 index 00000000..5e44b259 --- /dev/null +++ b/opensearch/config/opensearch.yml @@ -0,0 +1,19 @@ +## Cluster Settings +cluster.name: stac-cluster +node.name: os01 +network.host: 0.0.0.0 +transport.host: 0.0.0.0 +discovery.type: single-node +http.port: 9202 +http.cors.enabled: true +http.cors.allow-headers: X-Requested-With,Content-Type,Content-Length,Accept,Authorization + +path: + repo: + - /usr/share/opensearch/snapshots + +# Security +plugins.security.disabled: true +plugins.security.ssl.http.enabled: true + +node.max_local_storage_nodes: 3 From 5b9c8fb1856bcd8b0cc8b91227aadbfc8c439aa8 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 23 Jul 2025 20:28:48 -0400 Subject: [PATCH 2/7] opensearch docker compose --- Makefile | 96 -------------------------------------------------------- 1 file changed, 96 deletions(-) delete mode 100644 Makefile diff --git a/Makefile b/Makefile deleted file mode 100644 index db7e242a..00000000 --- a/Makefile +++ /dev/null @@ -1,96 +0,0 @@ -# STAC Auth Proxy Makefile -# Easily manage different backend configurations - -.PHONY: help up-pgstac up-opensearch up-all down clean logs ps - -# Default backend -BACKEND ?= pgstac - -help: - @echo "STAC Auth Proxy - Backend Management" - @echo "" - @echo "Available commands:" - @echo " make up-pgstac - Start with PostgreSQL/pgSTAC backend" - @echo " make up-opensearch - Start with OpenSearch backend" - @echo " make up-all - Start both backends (ports will conflict)" - @echo " make down - Stop all services" - @echo " make clean - Stop services and remove volumes" - @echo " make logs - View logs (use BACKEND=pgstac|opensearch)" - @echo " make ps - List running services" - @echo "" - @echo "Environment variables:" - @echo " STAC_PORT - STAC API port (default: 8001)" - @echo " UPSTREAM_URL - Proxy upstream URL (auto-configured)" - -# Start with pgSTAC backend -up-pgstac: - @echo "Starting with pgSTAC backend..." - @export COMPOSE_PROFILES=pgstac && \ - export UPSTREAM_URL=http://stac:8001 && \ - docker-compose up -d - @echo "" - @echo "Services started:" - @echo " STAC API (pgSTAC): http://localhost:8001" - @echo " Auth Proxy: http://localhost:8000" - @echo " Mock OIDC: http://localhost:8888" - @echo " PostgreSQL: localhost:5439" - -# Start with OpenSearch backend -up-opensearch: - @echo "Starting with OpenSearch backend..." - @export COMPOSE_PROFILES=opensearch && \ - export UPSTREAM_URL=http://stac-opensearch:8001 && \ - docker-compose up -d - @echo "" - @echo "Services started:" - @echo " STAC API (OpenSearch): http://localhost:8001" - @echo " Auth Proxy: http://localhost:8000" - @echo " Mock OIDC: http://localhost:8888" - @echo " OpenSearch: http://localhost:9200" - -# Start all services (for development/testing) -up-all: - @echo "Starting all services (both backends)..." - @echo "WARNING: Both STAC APIs will try to use port 8001" - @export COMPOSE_PROFILES=all && \ - docker-compose up -d - -# Stop all services -down: - docker-compose --profile pgstac --profile opensearch down - -# Clean up (stop and remove volumes) -clean: - docker-compose --profile pgstac --profile opensearch down -v - rm -rf .pgdata - -# View logs -logs: - @if [ "$(BACKEND)" = "opensearch" ]; then \ - docker-compose --profile opensearch logs -f stac-os database-os proxy; \ - else \ - docker-compose --profile pgstac logs -f stac-pg database-pg proxy; \ - fi - -# List running services -ps: - docker-compose --profile pgstac --profile opensearch ps - -# Test endpoints -test-stac: - @echo "Testing STAC API..." - @curl -s http://localhost:8001 | jq . - @echo "" - @echo "Testing Auth Proxy..." - @curl -s http://localhost:8000 | jq . - -# Initialize pgSTAC database -init-pgstac: - @echo "Initializing pgSTAC database..." - docker-compose --profile pgstac exec database-pg psql -U username -d postgis -c "CREATE EXTENSION IF NOT EXISTS postgis;" - docker-compose --profile pgstac exec database-pg psql -U username -d postgis -c "CREATE EXTENSION IF NOT EXISTS btree_gist;" - -# Initialize OpenSearch index -init-opensearch: - @echo "Initializing OpenSearch indices..." - @echo "This would typically be done by the STAC API on first run" \ No newline at end of file From 3bd629090880f0f3c9d8f1b792d20ffcba3248cd Mon Sep 17 00:00:00 2001 From: James Date: Thu, 24 Jul 2025 09:25:06 -0400 Subject: [PATCH 3/7] readme --- README.md | 16 ++++++++++++++++ docker-compose.yaml | 9 +++------ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 01377435..bc3d8965 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,22 @@ python -m stac_auth_proxy uvicorn --factory stac_auth_proxy:create_app ``` +### Docker compose + +Run all of the services required to run the application locally including the a database, STAC API, and Mock OICD provider using Docker compose. +The application can be run with [stac-fastapi-pgstac](https://github.com/stac-utils/stac-fastapi-pgstac) or with [stac-fastapi-elasticsearch-opensearch](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch) + +Spin up the application stack for Opensearch with +```sh +UPSTREAM_URL=http://stac-os:8001 docker compose --profile os up +``` + +and for pgstac with +```sh +UPSTREAM_URL=http://stac-pg:8001 docker compose --profile pg up +``` + + ### Installation For local development, we use [`uv`](https://docs.astral.sh/uv/) to manage project dependencies and environment. diff --git a/docker-compose.yaml b/docker-compose.yaml index 9abfe2b5..34f0162f 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -11,8 +11,8 @@ services: POSTGRES_USER: username POSTGRES_PASS: password POSTGRES_DBNAME: postgis - POSTGRES_HOST_READER: database - POSTGRES_HOST_WRITER: database + POSTGRES_HOST_READER: database-pg + POSTGRES_HOST_WRITER: database-pg POSTGRES_PORT: 5432 WEB_CONCURRENCY: 10 VSI_CACHE: TRUE @@ -58,6 +58,7 @@ services: database-pg: profiles: ["pg"] + container_name: database-pg image: ghcr.io/stac-utils/pgstac:v0.9.5 environment: POSTGRES_USER: username @@ -114,7 +115,3 @@ services: PORT: 8888 ports: - "8888:8888" - -networks: - eoapi-network: - name: eoapi-network From aa1bde55720ff03abc9de380610367e202278692 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 24 Jul 2025 09:35:19 -0400 Subject: [PATCH 4/7] readme --- README.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index bc3d8965..f6c6dc58 100644 --- a/README.md +++ b/README.md @@ -48,17 +48,16 @@ uvicorn --factory stac_auth_proxy:create_app ### Docker compose -Run all of the services required to run the application locally including the a database, STAC API, and Mock OICD provider using Docker compose. -The application can be run with [stac-fastapi-pgstac](https://github.com/stac-utils/stac-fastapi-pgstac) or with [stac-fastapi-elasticsearch-opensearch](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch) +Run all of the services required to run the application locally including the the database, STAC API, and Mock OICD provider using Docker compose. -Spin up the application stack for Opensearch with +Spin up the application stack with the pgSTAC backend using [stac-fastapi-pgstac](https://github.com/stac-utils/stac-fastapi-pgstac): ```sh -UPSTREAM_URL=http://stac-os:8001 docker compose --profile os up +UPSTREAM_URL=http://stac-pg:8001 docker compose --profile pg up ``` -and for pgstac with +and with the OpenSearch backend using [stac-fastapi-elasticsearch-opensearch](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch): ```sh -UPSTREAM_URL=http://stac-pg:8001 docker compose --profile pg up +UPSTREAM_URL=http://stac-os:8001 docker compose --profile os up ``` From cc0bac4afcbce5796f573ed914faed0311f4348d Mon Sep 17 00:00:00 2001 From: James Date: Thu, 24 Jul 2025 09:39:12 -0400 Subject: [PATCH 5/7] gitignore --- .gitignore | 3 +++ opensearch/config/opensearch.yml | 19 ------------------- 2 files changed, 3 insertions(+), 19 deletions(-) delete mode 100644 opensearch/config/opensearch.yml diff --git a/.gitignore b/.gitignore index 6a2bce2f..22a76d91 100644 --- a/.gitignore +++ b/.gitignore @@ -161,3 +161,6 @@ cython_debug/ # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ .pgdata +opensearch/ + + diff --git a/opensearch/config/opensearch.yml b/opensearch/config/opensearch.yml deleted file mode 100644 index 5e44b259..00000000 --- a/opensearch/config/opensearch.yml +++ /dev/null @@ -1,19 +0,0 @@ -## Cluster Settings -cluster.name: stac-cluster -node.name: os01 -network.host: 0.0.0.0 -transport.host: 0.0.0.0 -discovery.type: single-node -http.port: 9202 -http.cors.enabled: true -http.cors.allow-headers: X-Requested-With,Content-Type,Content-Length,Accept,Authorization - -path: - repo: - - /usr/share/opensearch/snapshots - -# Security -plugins.security.disabled: true -plugins.security.ssl.http.enabled: true - -node.max_local_storage_nodes: 3 From cd32528e8fd0c6c595a27a86a56bb73e97ab9a00 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 24 Jul 2025 10:34:19 -0400 Subject: [PATCH 6/7] opensearch config --- .gitignore | 5 +---- docker-compose.yaml | 47 +++++++++++++++++++++++---------------------- 2 files changed, 25 insertions(+), 27 deletions(-) diff --git a/.gitignore b/.gitignore index 22a76d91..07c8e00c 100644 --- a/.gitignore +++ b/.gitignore @@ -160,7 +160,4 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ -.pgdata -opensearch/ - - +.pgdata \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index 34f0162f..1fa47c0d 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -34,27 +34,27 @@ services: image: ghcr.io/stac-utils/stac-fastapi-os:latest hostname: stac-os environment: - - STAC_FASTAPI_TITLE=stac-fastapi-opensearch - - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Opensearch backend - - STAC_FASTAPI_VERSION=6.0.0 - - STAC_FASTAPI_LANDING_PAGE_ID=stac-fastapi-opensearch - - APP_HOST=0.0.0.0 - - APP_PORT=8001 - - RELOAD=true - - ENVIRONMENT=local - - WEB_CONCURRENCY=10 - - ES_HOST=database-os - - ES_PORT=9202 - - ES_USE_SSL=false - - ES_VERIFY_CERTS=false - - BACKEND=opensearch - - STAC_FASTAPI_RATE_LIMIT=200/minute + STAC_FASTAPI_TITLEL: stac-fastapi-opensearch + STAC_FASTAPI_DESCRIPTION: A STAC FastAPI with an Opensearch backend + STAC_FASTAPI_VERSION: 6.0.0 + STAC_FASTAPI_LANDING_PAGE_ID: stac-fastapi-opensearch + APP_HOST: 0.0.0.0 + APP_PORT: 8001 + RELOAD: true + ENVIRONMENT: local + WEB_CONCURRENCY: 10 + ES_HOST: database-os + ES_PORT: 9200 + ES_USE_SSL: false + ES_VERIFY_CERTS: false + BACKEND: opensearch + STAC_FASTAPI_RATE_LIMIT: 200/minute ports: - "8001:8001" depends_on: - database-os command: - bash -c "./scripts/wait-for-it-es.sh database-os:9202 && python -m stac_fastapi.opensearch.app" + bash -c "./scripts/wait-for-it-es.sh database-os:9200 && python -m stac_fastapi.opensearch.app" database-pg: profiles: ["pg"] @@ -79,14 +79,15 @@ services: image: opensearchproject/opensearch:${OPENSEARCH_VERSION:-2.11.1} hostname: database-os environment: - - discovery.type=single-node - - plugins.security.disabled=true - - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m - volumes: - - ./opensearch/config/opensearch.yml:/usr/share/opensearch/config/opensearch.yml - - ./opensearch/snapshots:/usr/share/opensearch/snapshots + cluster.name: stac-cluster + node.name: os01 + http.port: 9200 + http.cors.allow-headers: X-Requested-With,Content-Type,Content-Length,Accept,Authorization + discovery.type: single-node + plugins.security.disabled: true + OPENSEARCH_JAVA_OPTS: -Xms512m -Xmx512m ports: - - "9202:9202" + - "9200:9200" proxy: profiles: ["pg", "os"] From 88c16a69c0437d5010b5f6dbdcb05ecf61b8a77d Mon Sep 17 00:00:00 2001 From: James Date: Thu, 24 Jul 2025 12:34:22 -0400 Subject: [PATCH 7/7] sfeos 6.1.0 --- docker-compose.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 1fa47c0d..98b169cf 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,5 +1,4 @@ services: -# PostGRES STAC API stac-pg: profiles: ["pg"] image: ghcr.io/stac-utils/stac-fastapi-pgstac:5.0.2 @@ -27,11 +26,10 @@ services: - database-pg command: bash -c "./scripts/wait-for-it.sh database-pg:5432 && python -m stac_fastapi.pgstac.app" -# Opensearch STAC API stac-os: profiles: ["os"] container_name: stac-fastapi-os - image: ghcr.io/stac-utils/stac-fastapi-os:latest + image: ghcr.io/stac-utils/stac-fastapi-os:v6.1.0 hostname: stac-os environment: STAC_FASTAPI_TITLEL: stac-fastapi-opensearch