diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b7276348..06dd77916 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,24 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Changed +### Fixed + +## [v4.0.0a2] - 2025-04-20 + +### Added +- Added support for high-performance direct response mode for both Elasticsearch and Opensearch backends, controlled by the `ENABLE_DIRECT_RESPONSE` environment variable. When enabled (`ENABLE_DIRECT_RESPONSE=true`), endpoints return Starlette Response objects directly, bypassing FastAPI's jsonable_encoder and Pydantic serialization for significantly improved performance on large search responses. **Note:** In this mode, all FastAPI dependencies (including authentication, custom status codes, and validation) are disabled for all routes. Default is `false` for safety. A warning is logged at startup if enabled. See [issue #347](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/issues/347) and [PR #359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359). +- Added robust tests for the `ENABLE_DIRECT_RESPONSE` environment variable, covering both Elasticsearch and OpenSearch backends. Tests gracefully handle missing backends by attempting to import both configs and skipping if neither is available. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) + +### Changed +- Updated test suite to use `httpx.ASGITransport(app=...)` for FastAPI app testing (removes deprecation warning). [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) +- Updated stac-fastapi parent libraries to 5.2.0. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) +- Migrated Elasticsearch index template creation from legacy `put_template` to composable `put_index_template` API in `database_logic.py`. This resolves deprecation warnings and ensures compatibility with Elasticsearch 7.x and 8.x. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) +- Updated all Pydantic models to use `ConfigDict` instead of class-based `Config` for Pydantic v2 compatibility. This resolves deprecation warnings and prepares for Pydantic v3. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) +- Migrated all Pydantic `@root_validator` validators to `@model_validator` for Pydantic v2 compatibility. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) +- Migrated startup event handling from deprecated `@app.on_event("startup")` to FastAPI's recommended lifespan context manager. This removes deprecation warnings and ensures compatibility with future FastAPI versions. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) +- Refactored all boolean environment variable parsing in both Elasticsearch and OpenSearch backends to use the shared `get_bool_env` utility. This ensures robust and consistent handling of environment variables such as `ES_USE_SSL`, `ES_HTTP_COMPRESS`, and `ES_VERIFY_CERTS` across both backends. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) + + ### Fixed ## [v4.0.0a1] - 2925-04-17 @@ -343,25 +361,26 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Use genexp in execute_search and get_all_collections to return results. - Added db_to_stac serializer to item_collection method in core.py. -[Unreleased]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v4.0.0a1...main -[v4.0.0a1]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v4.0.0a0...v4.0.0a1 -[v4.0.0a0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.2.5...v4.0.0a0 -[v3.2.5]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.2.4...v3.2.5 -[v3.2.4]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.2.3...v3.2.4 -[v3.2.3]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.2.2...v3.2.3 -[v3.2.2]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.2.1...v3.2.2 -[v3.2.1]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.2.0...v3.2.1 -[v3.2.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.1.0...v3.2.0 -[v3.1.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.0.0...v3.1.0 -[v3.0.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v2.4.1...v3.0.0 -[v2.4.1]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v2.4.0...v2.4.1 -[v2.4.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v2.3.0...v2.4.0 -[v2.3.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v2.2.0...v2.3.0 -[v2.2.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v2.1.0...v2.2.0 -[v2.1.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v2.0.0...v2.1.0 -[v2.0.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v1.1.0...v2.0.0 -[v1.1.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v1.0.0...v1.1.0 -[v1.0.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v0.3.0...v1.0.0 -[v0.3.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v0.2.0...v0.3.0 -[v0.2.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v0.1.0...v0.2.0 -[v0.1.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v0.1.0 +[Unreleased]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v4.0.0a2...main +[v4.0.0a2]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v4.0.0a1...v4.0.0a2 +[v4.0.0a1]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v4.0.0a0...v4.0.0a1 +[v4.0.0a0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v3.2.5...v4.0.0a0 +[v3.2.5]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v3.2.4...v3.2.5 +[v3.2.4]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v3.2.3...v3.2.4 +[v3.2.3]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v3.2.2...v3.2.3 +[v3.2.2]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v3.2.1...v3.2.2 +[v3.2.1]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v3.2.0...v3.2.1 +[v3.2.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v3.1.0...v3.2.0 +[v3.1.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v3.0.0...v3.1.0 +[v3.0.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v2.4.1...v3.0.0 +[v2.4.1]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v2.4.0...v2.4.1 +[v2.4.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v2.3.0...v2.4.0 +[v2.3.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v2.2.0...v2.3.0 +[v2.2.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v2.1.0...v2.2.0 +[v2.1.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v2.0.0...v2.1.0 +[v2.0.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v1.1.0...v2.0.0 +[v1.1.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v1.0.0...v1.1.0 +[v1.0.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v0.3.0...v1.0.0 +[v0.3.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v0.2.0...v0.3.0 +[v0.2.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v0.1.0...v0.2.0 +[v0.1.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v0.1.0 diff --git a/README.md b/README.md index d6e648f32..896db23f9 100644 --- a/README.md +++ b/README.md @@ -29,8 +29,18 @@ - There is [Postman](https://documenter.getpostman.com/view/12888943/2s8ZDSdRHA) documentation here for examples on how to run some of the API routes locally - after starting the elasticsearch backend via the compose.yml file. - The `/examples` folder shows an example of running stac-fastapi-elasticsearch from PyPI in docker without needing any code from the repository. There is also a Postman collection here that you can load into Postman for testing the API routes. -- For changes, see the [Changelog](CHANGELOG.md) -- We are always welcoming contributions. For the development notes: [Contributing](CONTRIBUTING.md) + +### Performance Note + +The `enable_direct_response` option is provided by the stac-fastapi core library (introduced in stac-fastapi 5.2.0) and is available in this project starting from v4.0.0. + +**You can now control this setting via the `ENABLE_DIRECT_RESPONSE` environment variable.** + +When enabled (`ENABLE_DIRECT_RESPONSE=true`), endpoints return Starlette Response objects directly, bypassing FastAPI's default serialization for improved performance. **However, all FastAPI dependencies (including authentication, custom status codes, and validation) are disabled for all routes.** + +This mode is best suited for public or read-only APIs where authentication and custom logic are not required. Default is `false` for safety. + +See: [issue #347](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/issues/347) ### To install from PyPI: @@ -74,8 +84,9 @@ If you wish to use a different version, put the following in a file named `.env` in the same directory you run Docker Compose from: ```shell -ELASTICSEARCH_VERSION=7.17.1 -OPENSEARCH_VERSION=2.11.0 +ELASTICSEARCH_VERSION=8.11.0 +OPENSEARCH_VERSION=2.11.1 +ENABLE_DIRECT_RESPONSE=false ``` The most recent Elasticsearch 7.x versions should also work. See the [opensearch-py docs](https://github.com/opensearch-project/opensearch-py/blob/main/COMPATIBILITY.md) for compatibility information. @@ -100,8 +111,9 @@ You can customize additional settings in your `.env` file: | `RELOAD` | Enable auto-reload for development. | `true` | Optional | | `STAC_FASTAPI_RATE_LIMIT` | API rate limit per client. | `200/minute` | Optional | | `BACKEND` | Tests-related variable | `elasticsearch` or `opensearch` based on the backend | Optional | -| `ELASTICSEARCH_VERSION` | ElasticSearch version | `7.17.1` | Optional | -| `OPENSEARCH_VERSION` | OpenSearch version | `2.11.0` | Optional | +| `ELASTICSEARCH_VERSION` | Version of Elasticsearch to use. | `8.11.0` | Optional | +| `ENABLE_DIRECT_RESPONSE` | Enable direct response for maximum performance (disables all FastAPI dependencies, including authentication, custom status codes, and validation) | `false` | Optional | +| `OPENSEARCH_VERSION` | OpenSearch version | `2.11.1` | Optional | > [!NOTE] > The variables `ES_HOST`, `ES_PORT`, `ES_USE_SSL`, and `ES_VERIFY_CERTS` apply to both Elasticsearch and OpenSearch backends, so there is no need to rename the key names to `OS_` even if you're using OpenSearch. diff --git a/compose.yml b/compose.yml index a66e584fc..8f982ccb5 100644 --- a/compose.yml +++ b/compose.yml @@ -9,7 +9,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-elasticsearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Elasticsearch backend - - STAC_FASTAPI_VERSION=4.0.0a1 + - STAC_FASTAPI_VERSION=4.0.0a2 - APP_HOST=0.0.0.0 - APP_PORT=8080 - RELOAD=true @@ -41,7 +41,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-opensearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Opensearch backend - - STAC_FASTAPI_VERSION=4.0.0a1 + - STAC_FASTAPI_VERSION=4.0.0a2 - APP_HOST=0.0.0.0 - APP_PORT=8082 - RELOAD=true diff --git a/examples/auth/compose.basic_auth.yml b/examples/auth/compose.basic_auth.yml index c3e069ec0..88e95fa0a 100644 --- a/examples/auth/compose.basic_auth.yml +++ b/examples/auth/compose.basic_auth.yml @@ -9,7 +9,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-elasticsearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Elasticsearch backend - - STAC_FASTAPI_VERSION=4.0.0a1 + - STAC_FASTAPI_VERSION=4.0.0a2 - APP_HOST=0.0.0.0 - APP_PORT=8080 - RELOAD=true @@ -42,7 +42,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-opensearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Opensearch backend - - STAC_FASTAPI_VERSION=4.0.0a1 + - STAC_FASTAPI_VERSION=4.0.0a2 - APP_HOST=0.0.0.0 - APP_PORT=8082 - RELOAD=true diff --git a/examples/auth/compose.oauth2.yml b/examples/auth/compose.oauth2.yml index ccd3bb1f0..3a295862f 100644 --- a/examples/auth/compose.oauth2.yml +++ b/examples/auth/compose.oauth2.yml @@ -9,7 +9,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-elasticsearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Elasticsearch backend - - STAC_FASTAPI_VERSION=4.0.0a1 + - STAC_FASTAPI_VERSION=4.0.0a2 - APP_HOST=0.0.0.0 - APP_PORT=8080 - RELOAD=true @@ -43,7 +43,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-opensearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Opensearch backend - - STAC_FASTAPI_VERSION=4.0.0a1 + - STAC_FASTAPI_VERSION=4.0.0a2 - APP_HOST=0.0.0.0 - APP_PORT=8082 - RELOAD=true diff --git a/examples/auth/compose.route_dependencies.yml b/examples/auth/compose.route_dependencies.yml index 0516fccd8..08576691e 100644 --- a/examples/auth/compose.route_dependencies.yml +++ b/examples/auth/compose.route_dependencies.yml @@ -9,7 +9,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-elasticsearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Elasticsearch backend - - STAC_FASTAPI_VERSION=4.0.0a1 + - STAC_FASTAPI_VERSION=4.0.0a2 - APP_HOST=0.0.0.0 - APP_PORT=8080 - RELOAD=true @@ -42,7 +42,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-opensearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Opensearch backend - - STAC_FASTAPI_VERSION=4.0.0a1 + - STAC_FASTAPI_VERSION=4.0.0a2 - APP_HOST=0.0.0.0 - APP_PORT=8082 - RELOAD=true diff --git a/examples/rate_limit/compose.rate_limit.yml b/examples/rate_limit/compose.rate_limit.yml index 3fa902ab2..7d4340fba 100644 --- a/examples/rate_limit/compose.rate_limit.yml +++ b/examples/rate_limit/compose.rate_limit.yml @@ -9,7 +9,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-elasticsearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Elasticsearch backend - - STAC_FASTAPI_VERSION=4.0.0a1 + - STAC_FASTAPI_VERSION=4.0.0a2 - APP_HOST=0.0.0.0 - APP_PORT=8080 - RELOAD=true @@ -42,7 +42,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-opensearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Opensearch backend - - STAC_FASTAPI_VERSION=4.0.0a1 + - STAC_FASTAPI_VERSION=4.0.0a2 - APP_HOST=0.0.0.0 - APP_PORT=8082 - RELOAD=true diff --git a/stac_fastapi/core/setup.py b/stac_fastapi/core/setup.py index adde5c82a..ddf786b65 100644 --- a/stac_fastapi/core/setup.py +++ b/stac_fastapi/core/setup.py @@ -10,9 +10,9 @@ "attrs>=23.2.0", "pydantic>=2.4.1,<3.0.0", "stac_pydantic~=3.1.0", - "stac-fastapi.api==5.1.1", - "stac-fastapi.extensions==5.1.1", - "stac-fastapi.types==5.1.1", + "stac-fastapi.api==5.2.0", + "stac-fastapi.extensions==5.2.0", + "stac-fastapi.types==5.2.0", "orjson~=3.9.0", "overrides~=7.4.0", "geojson-pydantic~=1.0.0", diff --git a/stac_fastapi/core/stac_fastapi/core/core.py b/stac_fastapi/core/stac_fastapi/core/core.py index 16197da33..3ac14efc5 100644 --- a/stac_fastapi/core/stac_fastapi/core/core.py +++ b/stac_fastapi/core/stac_fastapi/core/core.py @@ -334,7 +334,7 @@ async def item_collection( search=search, limit=limit, sort=None, - token=token, # type: ignore + token=token, collection_ids=[collection_id], ) @@ -633,7 +633,7 @@ async def post_search( items, maybe_count, next_token = await self.database.execute_search( search=search, limit=limit, - token=search_request.token, # type: ignore + token=search_request.token, sort=sort, collection_ids=search_request.collections, ) @@ -701,7 +701,10 @@ async def create_item( database=self.database, settings=self.settings ) processed_items = [ - bulk_client.preprocess_item(item, base_url, BulkTransactionMethod.INSERT) for item in item["features"] # type: ignore + bulk_client.preprocess_item( + item, base_url, BulkTransactionMethod.INSERT + ) + for item in item["features"] ] await self.database.bulk_async( diff --git a/stac_fastapi/core/stac_fastapi/core/extensions/query.py b/stac_fastapi/core/stac_fastapi/core/extensions/query.py index 3084cbf84..f6e0868d5 100644 --- a/stac_fastapi/core/stac_fastapi/core/extensions/query.py +++ b/stac_fastapi/core/stac_fastapi/core/extensions/query.py @@ -10,7 +10,7 @@ from types import DynamicClassAttribute from typing import Any, Callable, Dict, Optional -from pydantic import BaseModel, root_validator +from pydantic import BaseModel, model_validator from stac_pydantic.utils import AutoValueEnum from stac_fastapi.extensions.core.query import QueryExtension as QueryExtensionBase @@ -63,7 +63,7 @@ class QueryExtensionPostRequest(BaseModel): query: Optional[Dict[str, Dict[Operator, Any]]] = None - @root_validator(pre=True) + @model_validator(mode="before") def validate_query_fields(cls, values: Dict) -> Dict: """Validate query fields.""" ... diff --git a/stac_fastapi/core/stac_fastapi/core/utilities.py b/stac_fastapi/core/stac_fastapi/core/utilities.py index d8c695296..e7aafe67c 100644 --- a/stac_fastapi/core/stac_fastapi/core/utilities.py +++ b/stac_fastapi/core/stac_fastapi/core/utilities.py @@ -3,6 +3,8 @@ This module contains functions for transforming geospatial coordinates, such as converting bounding boxes to polygon representations. """ +import logging +import os from typing import Any, Dict, List, Optional, Set, Union from stac_fastapi.types.stac import Item @@ -10,6 +12,33 @@ MAX_LIMIT = 10000 +def get_bool_env(name: str, default: bool = False) -> bool: + """ + Retrieve a boolean value from an environment variable. + + Args: + name (str): The name of the environment variable. + default (bool, optional): The default value to use if the variable is not set or unrecognized. Defaults to False. + + Returns: + bool: The boolean value parsed from the environment variable. + """ + value = os.getenv(name, str(default).lower()) + true_values = ("true", "1", "yes", "y") + false_values = ("false", "0", "no", "n") + if value.lower() in true_values: + return True + elif value.lower() in false_values: + return False + else: + logger = logging.getLogger(__name__) + logger.warning( + f"Environment variable '{name}' has unrecognized value '{value}'. " + f"Expected one of {true_values + false_values}. Using default: {default}" + ) + return default + + def bbox2polygon(b0: float, b1: float, b2: float, b3: float) -> List[List[List[float]]]: """Transform a bounding box represented by its four coordinates `b0`, `b1`, `b2`, and `b3` into a polygon. diff --git a/stac_fastapi/core/stac_fastapi/core/version.py b/stac_fastapi/core/stac_fastapi/core/version.py index af49b95b2..2c71d558f 100644 --- a/stac_fastapi/core/stac_fastapi/core/version.py +++ b/stac_fastapi/core/stac_fastapi/core/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "4.0.0a1" +__version__ = "4.0.0a2" diff --git a/stac_fastapi/elasticsearch/setup.py b/stac_fastapi/elasticsearch/setup.py index 1377211b6..77158e440 100644 --- a/stac_fastapi/elasticsearch/setup.py +++ b/stac_fastapi/elasticsearch/setup.py @@ -6,7 +6,7 @@ desc = f.read() install_requires = [ - "stac-fastapi-core==4.0.0a1", + "stac-fastapi-core==4.0.0a2", "elasticsearch[async]~=8.18.0", "uvicorn~=0.23.0", "starlette>=0.35.0,<0.36.0", diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py index 9510eaa6e..91e239a49 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py @@ -87,7 +87,7 @@ api = StacApi( title=os.getenv("STAC_FASTAPI_TITLE", "stac-fastapi-elasticsearch"), description=os.getenv("STAC_FASTAPI_DESCRIPTION", "stac-fastapi-elasticsearch"), - api_version=os.getenv("STAC_FASTAPI_VERSION", "2.1"), + api_version=os.getenv("STAC_FASTAPI_VERSION", "4.0.0a2"), settings=settings, extensions=extensions, client=CoreClient( diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py index fb9e2e0f7..2044a4b2f 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py @@ -1,19 +1,22 @@ """API configuration.""" +import logging import os import ssl from typing import Any, Dict, Set import certifi +from elasticsearch._async.client import AsyncElasticsearch -from elasticsearch import AsyncElasticsearch, Elasticsearch # type: ignore +from elasticsearch import Elasticsearch # type: ignore[attr-defined] from stac_fastapi.core.base_settings import ApiBaseSettings +from stac_fastapi.core.utilities import get_bool_env from stac_fastapi.types.config import ApiSettings def _es_config() -> Dict[str, Any]: # Determine the scheme (http or https) - use_ssl = os.getenv("ES_USE_SSL", "true").lower() == "true" + use_ssl = get_bool_env("ES_USE_SSL", default=True) scheme = "https" if use_ssl else "http" # Configure the hosts parameter with the correct scheme @@ -44,7 +47,7 @@ def _es_config() -> Dict[str, Any]: config["headers"] = headers - http_compress = os.getenv("ES_HTTP_COMPRESS", "true").lower() == "true" + http_compress = get_bool_env("ES_HTTP_COMPRESS", default=True) if http_compress: config["http_compress"] = True @@ -53,8 +56,8 @@ def _es_config() -> Dict[str, Any]: return config # Include SSL settings if using https - config["ssl_version"] = ssl.TLSVersion.TLSv1_3 # type: ignore - config["verify_certs"] = os.getenv("ES_VERIFY_CERTS", "true").lower() != "false" # type: ignore + config["ssl_version"] = ssl.TLSVersion.TLSv1_3 + config["verify_certs"] = get_bool_env("ES_VERIFY_CERTS", default=True) # Include CA Certificates if verifying certs if config["verify_certs"]: @@ -71,11 +74,18 @@ def _es_config() -> Dict[str, Any]: class ElasticsearchSettings(ApiSettings, ApiBaseSettings): - """API settings.""" + """ + API settings. + + Set enable_direct_response via the ENABLE_DIRECT_RESPONSE environment variable. + If enabled, all API routes use direct response for maximum performance, but ALL FastAPI dependencies (including authentication, custom status codes, and validation) are disabled. + Default is False for safety. + """ - # Fields which are defined by STAC but not included in the database model forbidden_fields: Set[str] = _forbidden_fields indexed_fields: Set[str] = {"datetime"} + enable_response_models: bool = False + enable_direct_response: bool = get_bool_env("ENABLE_DIRECT_RESPONSE", default=False) @property def create_client(self): @@ -84,13 +94,31 @@ def create_client(self): class AsyncElasticsearchSettings(ApiSettings, ApiBaseSettings): - """API settings.""" + """ + API settings. + + Set enable_direct_response via the ENABLE_DIRECT_RESPONSE environment variable. + If enabled, all API routes use direct response for maximum performance, but ALL FastAPI dependencies (including authentication, custom status codes, and validation) are disabled. + Default is False for safety. + """ - # Fields which are defined by STAC but not included in the database model forbidden_fields: Set[str] = _forbidden_fields indexed_fields: Set[str] = {"datetime"} + enable_response_models: bool = False + enable_direct_response: bool = get_bool_env("ENABLE_DIRECT_RESPONSE", default=False) @property def create_client(self): """Create async elasticsearch client.""" return AsyncElasticsearch(**_es_config()) + + +# Warn at import if direct response is enabled (applies to either settings class) +if ( + ElasticsearchSettings().enable_direct_response + or AsyncElasticsearchSettings().enable_direct_response +): + logging.basicConfig(level=logging.WARNING) + logging.warning( + "ENABLE_DIRECT_RESPONSE is True: All FastAPI dependencies (including authentication) are DISABLED for all routes!" + ) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index ec84de574..f57ef9bb2 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -8,10 +8,11 @@ from typing import Any, Dict, Iterable, List, Optional, Tuple, Type import attr +import elasticsearch.helpers as helpers from elasticsearch.dsl import Q, Search +from elasticsearch.exceptions import NotFoundError as ESNotFoundError from starlette.requests import Request -from elasticsearch import exceptions, helpers # type: ignore from stac_fastapi.core.base_database_logic import BaseDatabaseLogic from stac_fastapi.core.database_logic import ( COLLECTIONS_INDEX, @@ -50,19 +51,18 @@ async def create_index_templates() -> None: """ client = AsyncElasticsearchSettings().create_client - await client.indices.put_template( + await client.indices.put_index_template( name=f"template_{COLLECTIONS_INDEX}", body={ "index_patterns": [f"{COLLECTIONS_INDEX}*"], - "mappings": ES_COLLECTIONS_MAPPINGS, + "template": {"mappings": ES_COLLECTIONS_MAPPINGS}, }, ) - await client.indices.put_template( + await client.indices.put_index_template( name=f"template_{ITEMS_INDEX_PREFIX}", body={ "index_patterns": [f"{ITEMS_INDEX_PREFIX}*"], - "settings": ES_ITEMS_SETTINGS, - "mappings": ES_ITEMS_MAPPINGS, + "template": {"settings": ES_ITEMS_SETTINGS, "mappings": ES_ITEMS_MAPPINGS}, }, ) await client.close() @@ -80,7 +80,7 @@ async def create_collection_index() -> None: await client.options(ignore_status=400).indices.create( index=f"{COLLECTIONS_INDEX}-000001", - aliases={COLLECTIONS_INDEX: {}}, + body={"aliases": {COLLECTIONS_INDEX: {}}}, ) await client.close() @@ -100,7 +100,7 @@ async def create_item_index(collection_id: str): await client.options(ignore_status=400).indices.create( index=f"{index_by_collection_id(collection_id)}-000001", - aliases={index_alias_by_collection_id(collection_id): {}}, + body={"aliases": {index_alias_by_collection_id(collection_id): {}}}, ) await client.close() @@ -272,7 +272,7 @@ async def get_one_item(self, collection_id: str, item_id: str) -> Dict: index=index_alias_by_collection_id(collection_id), id=mk_item_id(item_id, collection_id), ) - except exceptions.NotFoundError: + except ESNotFoundError: raise NotFoundError( f"Item {item_id} does not exist inside Collection {collection_id}" ) @@ -512,7 +512,7 @@ async def execute_search( try: es_response = await search_task - except exceptions.NotFoundError: + except ESNotFoundError: raise NotFoundError(f"Collections '{collection_ids}' do not exist") hits = es_response["hits"]["hits"] @@ -595,7 +595,7 @@ def _fill_aggregation_parameters(name: str, agg: dict) -> dict: try: db_response = await search_task - except exceptions.NotFoundError: + except ESNotFoundError: raise NotFoundError(f"Collections '{collection_ids}' do not exist") return db_response @@ -721,7 +721,7 @@ async def delete_item( id=mk_item_id(item_id, collection_id), refresh=refresh, ) - except exceptions.NotFoundError: + except ESNotFoundError: raise NotFoundError( f"Item {item_id} in collection {collection_id} not found" ) @@ -741,7 +741,7 @@ async def get_items_mapping(self, collection_id: str) -> Dict[str, Any]: index=index_name, allow_no_indices=False ) return mapping.body - except exceptions.NotFoundError: + except ESNotFoundError: raise NotFoundError(f"Mapping for index {index_name} not found") async def create_collection(self, collection: Collection, refresh: bool = False): @@ -792,7 +792,7 @@ async def find_collection(self, collection_id: str) -> Collection: collection = await self.client.get( index=COLLECTIONS_INDEX, id=collection_id ) - except exceptions.NotFoundError: + except ESNotFoundError: raise NotFoundError(f"Collection {collection_id} not found") return collection["_source"] diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py index af49b95b2..2c71d558f 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "4.0.0a1" +__version__ = "4.0.0a2" diff --git a/stac_fastapi/opensearch/setup.py b/stac_fastapi/opensearch/setup.py index ece68679f..4d7187330 100644 --- a/stac_fastapi/opensearch/setup.py +++ b/stac_fastapi/opensearch/setup.py @@ -6,7 +6,7 @@ desc = f.read() install_requires = [ - "stac-fastapi-core==4.0.0a1", + "stac-fastapi-core==4.0.0a2", "opensearch-py~=2.8.0", "opensearch-py[async]~=2.8.0", "uvicorn~=0.23.0", diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py index 900383021..504d5eab2 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py @@ -87,7 +87,7 @@ api = StacApi( title=os.getenv("STAC_FASTAPI_TITLE", "stac-fastapi-opensearch"), description=os.getenv("STAC_FASTAPI_DESCRIPTION", "stac-fastapi-opensearch"), - api_version=os.getenv("STAC_FASTAPI_VERSION", "2.1"), + api_version=os.getenv("STAC_FASTAPI_VERSION", "4.0.0a2"), settings=settings, extensions=extensions, client=CoreClient( @@ -100,6 +100,7 @@ app = api.app app.root_path = os.getenv("STAC_FASTAPI_ROOT_PATH", "") + # Add rate limit setup_rate_limit(app, rate_limit=os.getenv("STAC_FASTAPI_RATE_LIMIT")) diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py index 6de2ab915..004984682 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py @@ -1,4 +1,5 @@ """API configuration.""" +import logging import os import ssl from typing import Any, Dict, Set @@ -7,12 +8,13 @@ from opensearchpy import AsyncOpenSearch, OpenSearch from stac_fastapi.core.base_settings import ApiBaseSettings +from stac_fastapi.core.utilities import get_bool_env from stac_fastapi.types.config import ApiSettings def _es_config() -> Dict[str, Any]: # Determine the scheme (http or https) - use_ssl = os.getenv("ES_USE_SSL", "true").lower() == "true" + use_ssl = get_bool_env("ES_USE_SSL", default=True) scheme = "https" if use_ssl else "http" # Configure the hosts parameter with the correct scheme @@ -33,7 +35,7 @@ def _es_config() -> Dict[str, Any]: "headers": {"accept": "application/json", "Content-Type": "application/json"}, } - http_compress = os.getenv("ES_HTTP_COMPRESS", "true").lower() == "true" + http_compress = get_bool_env("ES_HTTP_COMPRESS", default=True) if http_compress: config["http_compress"] = True @@ -42,8 +44,8 @@ def _es_config() -> Dict[str, Any]: return config # Include SSL settings if using https - config["ssl_version"] = ssl.PROTOCOL_SSLv23 # type: ignore - config["verify_certs"] = os.getenv("ES_VERIFY_CERTS", "true").lower() != "false" # type: ignore + config["ssl_version"] = ssl.PROTOCOL_SSLv23 + config["verify_certs"] = get_bool_env("ES_VERIFY_CERTS", default=True) # Include CA Certificates if verifying certs if config["verify_certs"]: @@ -69,11 +71,18 @@ def _es_config() -> Dict[str, Any]: class OpensearchSettings(ApiSettings, ApiBaseSettings): - """API settings.""" + """ + API settings. + + Set enable_direct_response via the ENABLE_DIRECT_RESPONSE environment variable. + If enabled, all API routes use direct response for maximum performance, but ALL FastAPI dependencies (including authentication, custom status codes, and validation) are disabled. + Default is False for safety. + """ - # Fields which are defined by STAC but not included in the database model forbidden_fields: Set[str] = _forbidden_fields indexed_fields: Set[str] = {"datetime"} + enable_response_models: bool = False + enable_direct_response: bool = get_bool_env("ENABLE_DIRECT_RESPONSE", default=False) @property def create_client(self): @@ -82,13 +91,31 @@ def create_client(self): class AsyncOpensearchSettings(ApiSettings, ApiBaseSettings): - """API settings.""" + """ + API settings. + + Set enable_direct_response via the ENABLE_DIRECT_RESPONSE environment variable. + If enabled, all API routes use direct response for maximum performance, but ALL FastAPI dependencies (including authentication, custom status codes, and validation) are disabled. + Default is False for safety. + """ - # Fields which are defined by STAC but not included in the database model forbidden_fields: Set[str] = _forbidden_fields indexed_fields: Set[str] = {"datetime"} + enable_response_models: bool = False + enable_direct_response: bool = get_bool_env("ENABLE_DIRECT_RESPONSE", default=False) @property def create_client(self): """Create async elasticsearch client.""" return AsyncOpenSearch(**_es_config()) + + +# Warn at import if direct response is enabled (applies to either settings class) +if ( + OpensearchSettings().enable_direct_response + or AsyncOpensearchSettings().enable_direct_response +): + logging.basicConfig(level=logging.WARNING) + logging.warning( + "ENABLE_DIRECT_RESPONSE is True: All FastAPI dependencies (including authentication) are DISABLED for all routes!" + ) diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py index 22e6ffe00..3184fa066 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py @@ -9,7 +9,6 @@ import attr from opensearchpy import exceptions, helpers -from opensearchpy.exceptions import TransportError from opensearchpy.helpers.query import Q from opensearchpy.helpers.search import Search from starlette.requests import Request @@ -80,24 +79,21 @@ async def create_collection_index() -> None: """ client = AsyncSearchSettings().create_client - search_body: Dict[str, Any] = { - "aliases": {COLLECTIONS_INDEX: {}}, - } - index = f"{COLLECTIONS_INDEX}-000001" - try: - await client.indices.create(index=index, body=search_body) - except TransportError as e: - if e.status_code == 400: - pass # Ignore 400 status codes - else: - raise e - + exists = await client.indices.exists(index=index) + if not exists: + await client.indices.create( + index=index, + body={ + "aliases": {COLLECTIONS_INDEX: {}}, + "mappings": ES_COLLECTIONS_MAPPINGS, + }, + ) await client.close() -async def create_item_index(collection_id: str): +async def create_item_index(collection_id: str) -> None: """ Create the index for Items. The settings of the index template will be used implicitly. @@ -109,24 +105,22 @@ async def create_item_index(collection_id: str): """ client = AsyncSearchSettings().create_client - search_body: Dict[str, Any] = { - "aliases": {index_alias_by_collection_id(collection_id): {}}, - } - try: + index_name = f"{index_by_collection_id(collection_id)}-000001" + exists = await client.indices.exists(index=index_name) + if not exists: await client.indices.create( - index=f"{index_by_collection_id(collection_id)}-000001", body=search_body + index=index_name, + body={ + "aliases": {index_alias_by_collection_id(collection_id): {}}, + "mappings": ES_ITEMS_MAPPINGS, + "settings": ES_ITEMS_SETTINGS, + }, ) - except TransportError as e: - if e.status_code == 400: - pass # Ignore 400 status codes - else: - raise e - await client.close() -async def delete_item_index(collection_id: str): +async def delete_item_index(collection_id: str) -> None: """Delete the index for items in a collection. Args: diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py index af49b95b2..2c71d558f 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "4.0.0a1" +__version__ = "4.0.0a2" diff --git a/stac_fastapi/tests/api/test_api.py b/stac_fastapi/tests/api/test_api.py index 645458075..fb128f746 100644 --- a/stac_fastapi/tests/api/test_api.py +++ b/stac_fastapi/tests/api/test_api.py @@ -7,6 +7,7 @@ ROUTES = { "GET /_mgmt/ping", + "GET /_mgmt/health", "GET /docs/oauth2-redirect", "HEAD /docs/oauth2-redirect", "GET /", diff --git a/stac_fastapi/tests/conftest.py b/stac_fastapi/tests/conftest.py index 651cdadb5..a82f14853 100644 --- a/stac_fastapi/tests/conftest.py +++ b/stac_fastapi/tests/conftest.py @@ -8,7 +8,8 @@ import pytest import pytest_asyncio from fastapi import Depends, HTTPException, security, status -from httpx import AsyncClient +from httpx import ASGITransport, AsyncClient +from pydantic import ConfigDict from stac_pydantic import api from stac_fastapi.api.app import StacApi @@ -85,8 +86,7 @@ def __init__( class TestSettings(AsyncSettings): - class Config: - env_file = ".env.test" + model_config = ConfigDict(env_file=".env.test") settings = TestSettings() @@ -243,7 +243,9 @@ async def app_client(app): await create_index_templates() await create_collection_index() - async with AsyncClient(app=app, base_url="http://test-server") as c: + async with AsyncClient( + transport=ASGITransport(app=app), base_url="http://test-server" + ) as c: yield c @@ -302,7 +304,9 @@ async def app_client_rate_limit(app_rate_limit): await create_index_templates() await create_collection_index() - async with AsyncClient(app=app_rate_limit, base_url="http://test-server") as c: + async with AsyncClient( + transport=ASGITransport(app=app_rate_limit), base_url="http://test-server" + ) as c: yield c @@ -392,7 +396,9 @@ async def app_client_basic_auth(app_basic_auth): await create_index_templates() await create_collection_index() - async with AsyncClient(app=app_basic_auth, base_url="http://test-server") as c: + async with AsyncClient( + transport=ASGITransport(app=app_basic_auth), base_url="http://test-server" + ) as c: yield c @@ -469,6 +475,7 @@ async def route_dependencies_client(route_dependencies_app): await create_collection_index() async with AsyncClient( - app=route_dependencies_app, base_url="http://test-server" + transport=ASGITransport(app=route_dependencies_app), + base_url="http://test-server", ) as c: yield c diff --git a/stac_fastapi/tests/elasticsearch/test_direct_response.py b/stac_fastapi/tests/elasticsearch/test_direct_response.py new file mode 100644 index 000000000..bbbceb560 --- /dev/null +++ b/stac_fastapi/tests/elasticsearch/test_direct_response.py @@ -0,0 +1,39 @@ +import importlib + +import pytest + + +def get_settings_class(): + """ + Try to import ElasticsearchSettings or OpenSearchSettings, whichever is available. + Returns a tuple: (settings_class, config_module) + """ + try: + config = importlib.import_module("stac_fastapi.elasticsearch.config") + importlib.reload(config) + return config.ElasticsearchSettings, config + except ModuleNotFoundError: + try: + config = importlib.import_module("stac_fastapi.opensearch.config") + importlib.reload(config) + return config.OpensearchSettings, config + except ModuleNotFoundError: + pytest.skip( + "Neither Elasticsearch nor OpenSearch config module is available." + ) + + +def test_enable_direct_response_true(monkeypatch): + """Test that ENABLE_DIRECT_RESPONSE env var enables direct response config.""" + monkeypatch.setenv("ENABLE_DIRECT_RESPONSE", "true") + settings_class, _ = get_settings_class() + settings = settings_class() + assert settings.enable_direct_response is True + + +def test_enable_direct_response_false(monkeypatch): + """Test that ENABLE_DIRECT_RESPONSE env var disables direct response config.""" + monkeypatch.setenv("ENABLE_DIRECT_RESPONSE", "false") + settings_class, _ = get_settings_class() + settings = settings_class() + assert settings.enable_direct_response is False