diff --git a/CHANGELOG.md b/CHANGELOG.md index 283907e73..24d0e6fd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,15 +12,19 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - `STAC_INDEX_ASSETS` environment variable to allow asset serialization to be configurable. [#433](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/433) - Added the `ENV_MAX_LIMIT` environment variable to SFEOS, allowing overriding of the `MAX_LIMIT`, which controls the `?limit` parameter for returned items and STAC collections. [#434](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/434) -- Updated the `format_datetime_range` function to support milliseconds. [#423](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/423) +- Sort, Query, and Filter extension and functionality to the item collection route. [#437](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/437) ### Changed -- Changed assets serialization to prevent mapping explosion while allowing asset inforamtion to be indexed. [#341](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/341) +- Changed assets serialization to prevent mapping explosion while allowing asset information to be indexed. [#341](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/341) +- Simplified the item_collection function in core.py, moving the request to the get_search function. [#437](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/437) +- Updated the `format_datetime_range` function to support milliseconds. [#423](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/423) - Blocked the /collections/{collection_id}/bulk_items endpoint when environmental variable ENABLE_DATETIME_INDEX_FILTERING is set to true. [#438](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/438) ### Fixed +- Fixed issue where sortby was not accepting the default sort, where a + or - was not specified before the field value ie. localhost:8081/collections/{collection_id}/items?sortby=id. [#437](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/437) + ## [v6.2.1] - 2025-09-02 ### Added diff --git a/dockerfiles/Dockerfile.dev.os b/dockerfiles/Dockerfile.dev.os index a544e94af..a7fc113d6 100644 --- a/dockerfiles/Dockerfile.dev.os +++ b/dockerfiles/Dockerfile.dev.os @@ -4,11 +4,10 @@ FROM python:3.10-slim # update apt pkgs, and install build-essential for ciso8601 RUN apt-get update && \ apt-get -y upgrade && \ - apt-get -y install build-essential && \ + apt-get -y install build-essential git && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -RUN apt-get -y install git # update certs used by Requests ENV CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt diff --git a/stac_fastapi/core/stac_fastapi/core/core.py b/stac_fastapi/core/stac_fastapi/core/core.py index 82d03894c..7b4502669 100644 --- a/stac_fastapi/core/stac_fastapi/core/core.py +++ b/stac_fastapi/core/stac_fastapi/core/core.py @@ -284,86 +284,60 @@ async def get_collection( async def item_collection( self, collection_id: str, + request: Request, bbox: Optional[BBox] = None, datetime: Optional[str] = None, limit: Optional[int] = None, + sortby: Optional[str] = None, + filter_expr: Optional[str] = None, + filter_lang: Optional[str] = None, token: Optional[str] = None, + query: Optional[str] = None, **kwargs, ) -> stac_types.ItemCollection: - """Read items from a specific collection in the database. + """List items within a specific collection. + + This endpoint delegates to ``get_search`` under the hood with + ``collections=[collection_id]`` so that filtering, sorting and pagination + behave identically to the Search endpoints. Args: - collection_id (str): The identifier of the collection to read items from. - bbox (Optional[BBox]): The bounding box to filter items by. - datetime (Optional[str]): The datetime range to filter items by. - limit (int): The maximum number of items to return. - token (str): A token used for pagination. - request (Request): The incoming request. + collection_id (str): ID of the collection to list items from. + request (Request): FastAPI Request object. + bbox (Optional[BBox]): Optional bounding box filter. + datetime (Optional[str]): Optional datetime or interval filter. + limit (Optional[int]): Optional page size. Defaults to env ``STAC_ITEM_LIMIT`` when unset. + sortby (Optional[str]): Optional sort specification. Accepts repeated values + like ``sortby=-properties.datetime`` or ``sortby=+id``. Bare fields (e.g. ``sortby=id``) + imply ascending order. + token (Optional[str]): Optional pagination token. + query (Optional[str]): Optional query string. + filter_expr (Optional[str]): Optional filter expression. + filter_lang (Optional[str]): Optional filter language. Returns: - ItemCollection: An `ItemCollection` object containing the items from the specified collection that meet - the filter criteria and links to various resources. + ItemCollection: Feature collection with items, paging links, and counts. Raises: - HTTPException: If the specified collection is not found. - Exception: If any error occurs while reading the items from the database. + HTTPException: 404 if the collection does not exist. """ - request: Request = kwargs["request"] - token = request.query_params.get("token") - - base_url = str(request.base_url) - - collection = await self.get_collection( - collection_id=collection_id, request=request - ) - collection_id = collection.get("id") - if collection_id is None: - raise HTTPException(status_code=404, detail="Collection not found") - - search = self.database.make_search() - search = self.database.apply_collections_filter( - search=search, collection_ids=[collection_id] - ) - try: - search, datetime_search = self.database.apply_datetime_filter( - search=search, datetime=datetime - ) - except (ValueError, TypeError) as e: - # Handle invalid interval formats if return_date fails - msg = f"Invalid interval format: {datetime}, error: {e}" - logger.error(msg) - raise HTTPException(status_code=400, detail=msg) - - if bbox: - bbox = [float(x) for x in bbox] - if len(bbox) == 6: - bbox = [bbox[0], bbox[1], bbox[3], bbox[4]] - - search = self.database.apply_bbox_filter(search=search, bbox=bbox) + await self.get_collection(collection_id=collection_id, request=request) + except Exception: + raise HTTPException(status_code=404, detail="Collection not found") - limit = int(request.query_params.get("limit", os.getenv("STAC_ITEM_LIMIT", 10))) - items, maybe_count, next_token = await self.database.execute_search( - search=search, + # Delegate directly to GET search for consistency + return await self.get_search( + request=request, + collections=[collection_id], + bbox=bbox, + datetime=datetime, limit=limit, - sort=None, token=token, - collection_ids=[collection_id], - datetime_search=datetime_search, - ) - - items = [ - self.item_serializer.db_to_stac(item, base_url=base_url) for item in items - ] - - links = await PagingLinks(request=request, next=next_token).get_links() - - return stac_types.ItemCollection( - type="FeatureCollection", - features=items, - links=links, - numReturned=len(items), - numMatched=maybe_count, + sortby=sortby, + query=query, + filter_expr=filter_expr, + filter_lang=filter_lang, ) async def get_item( @@ -429,6 +403,7 @@ async def get_search( HTTPException: If any error occurs while searching the catalog. """ limit = int(request.query_params.get("limit", os.getenv("STAC_ITEM_LIMIT", 10))) + base_args = { "collections": collections, "ids": ids, @@ -446,10 +421,18 @@ async def get_search( base_args["intersects"] = orjson.loads(unquote_plus(intersects)) if sortby: - base_args["sortby"] = [ - {"field": sort[1:], "direction": "desc" if sort[0] == "-" else "asc"} - for sort in sortby - ] + parsed_sort = [] + for raw in sortby: + if not isinstance(raw, str): + continue + s = raw.strip() + if not s: + continue + direction = "desc" if s[0] == "-" else "asc" + field = s[1:] if s and s[0] in "+-" else s + parsed_sort.append({"field": field, "direction": direction}) + if parsed_sort: + base_args["sortby"] = parsed_sort if filter_expr: base_args["filter_lang"] = "cql2-json" @@ -526,13 +509,15 @@ async def post_search( search = self.database.apply_bbox_filter(search=search, bbox=bbox) - if search_request.intersects: + if hasattr(search_request, "intersects") and getattr( + search_request, "intersects" + ): search = self.database.apply_intersects_filter( - search=search, intersects=search_request.intersects + search=search, intersects=getattr(search_request, "intersects") ) - if search_request.query: - for field_name, expr in search_request.query.items(): + if hasattr(search_request, "query") and getattr(search_request, "query"): + for field_name, expr in getattr(search_request, "query").items(): field = "properties__" + field_name for op, value in expr.items(): # Convert enum to string @@ -541,9 +526,14 @@ async def post_search( search=search, op=operator, field=field, value=value ) - # only cql2_json is supported here + # Apply CQL2 filter (support both 'filter_expr' and canonical 'filter') + cql2_filter = None if hasattr(search_request, "filter_expr"): cql2_filter = getattr(search_request, "filter_expr", None) + if cql2_filter is None and hasattr(search_request, "filter"): + cql2_filter = getattr(search_request, "filter", None) + + if cql2_filter is not None: try: search = await self.database.apply_cql2_filter(search, cql2_filter) except Exception as e: @@ -561,19 +551,23 @@ async def post_search( ) sort = None - if search_request.sortby: - sort = self.database.populate_sort(search_request.sortby) + if hasattr(search_request, "sortby") and getattr(search_request, "sortby"): + sort = self.database.populate_sort(getattr(search_request, "sortby")) limit = 10 if search_request.limit: limit = search_request.limit + # Use token from the request if the model doesn't define it + token_param = getattr( + search_request, "token", None + ) or request.query_params.get("token") items, maybe_count, next_token = await self.database.execute_search( search=search, limit=limit, - token=search_request.token, + token=token_param, sort=sort, - collection_ids=search_request.collections, + collection_ids=getattr(search_request, "collections", None), datetime_search=datetime_search, ) @@ -917,7 +911,7 @@ async def delete_collection(self, collection_id: str, **kwargs) -> None: @attr.s class BulkTransactionsClient(BaseBulkTransactionsClient): - """A client for posting bulk transactions to a Postgres database. + """A client for posting bulk transactions. Attributes: session: An instance of `Session` to use for database connection. diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py index b0fbbd6b9..34c582c19 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py @@ -7,7 +7,12 @@ from fastapi import FastAPI from stac_fastapi.api.app import StacApi -from stac_fastapi.api.models import create_get_request_model, create_post_request_model +from stac_fastapi.api.models import ( + ItemCollectionUri, + create_get_request_model, + create_post_request_model, + create_request_model, +) from stac_fastapi.core.core import ( BulkTransactionsClient, CoreClient, @@ -39,6 +44,8 @@ TransactionExtension, ) from stac_fastapi.extensions.core.filter import FilterConformanceClasses +from stac_fastapi.extensions.core.query import QueryConformanceClasses +from stac_fastapi.extensions.core.sort import SortConformanceClasses from stac_fastapi.extensions.third_party import BulkTransactionExtension from stac_fastapi.sfeos_helpers.aggregation import EsAsyncBaseAggregationClient from stac_fastapi.sfeos_helpers.filter import EsAsyncBaseFiltersClient @@ -54,6 +61,7 @@ database_logic = DatabaseLogic() + filter_extension = FilterExtension( client=EsAsyncBaseFiltersClient(database=database_logic) ) @@ -114,6 +122,21 @@ post_request_model = create_post_request_model(search_extensions) +items_get_request_model = create_request_model( + model_name="ItemCollectionUri", + base_model=ItemCollectionUri, + extensions=[ + SortExtension( + conformance_classes=[SortConformanceClasses.ITEMS], + ), + QueryExtension( + conformance_classes=[QueryConformanceClasses.ITEMS], + ), + filter_extension, + ], + request_type="GET", +) + app_config = { "title": os.getenv("STAC_FASTAPI_TITLE", "stac-fastapi-elasticsearch"), "description": os.getenv("STAC_FASTAPI_DESCRIPTION", "stac-fastapi-elasticsearch"), @@ -128,6 +151,7 @@ ), "search_get_request_model": create_get_request_model(search_extensions), "search_post_request_model": post_request_model, + "items_get_request_model": items_get_request_model, "route_dependencies": get_route_dependencies(), } diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py index 1fa036d1b..26b4bc0ca 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py @@ -7,7 +7,12 @@ from fastapi import FastAPI from stac_fastapi.api.app import StacApi -from stac_fastapi.api.models import create_get_request_model, create_post_request_model +from stac_fastapi.api.models import ( + ItemCollectionUri, + create_get_request_model, + create_post_request_model, + create_request_model, +) from stac_fastapi.core.core import ( BulkTransactionsClient, CoreClient, @@ -33,6 +38,8 @@ TransactionExtension, ) from stac_fastapi.extensions.core.filter import FilterConformanceClasses +from stac_fastapi.extensions.core.query import QueryConformanceClasses +from stac_fastapi.extensions.core.sort import SortConformanceClasses from stac_fastapi.extensions.third_party import BulkTransactionExtension from stac_fastapi.opensearch.config import OpensearchSettings from stac_fastapi.opensearch.database_logic import ( @@ -115,6 +122,21 @@ post_request_model = create_post_request_model(search_extensions) +items_get_request_model = create_request_model( + model_name="ItemCollectionUri", + base_model=ItemCollectionUri, + extensions=[ + SortExtension( + conformance_classes=[SortConformanceClasses.ITEMS], + ), + QueryExtension( + conformance_classes=[QueryConformanceClasses.ITEMS], + ), + filter_extension, + ], + request_type="GET", +) + app_config = { "title": os.getenv("STAC_FASTAPI_TITLE", "stac-fastapi-opensearch"), "description": os.getenv("STAC_FASTAPI_DESCRIPTION", "stac-fastapi-opensearch"), @@ -129,6 +151,7 @@ ), "search_get_request_model": create_get_request_model(search_extensions), "search_post_request_model": post_request_model, + "items_get_request_model": items_get_request_model, "route_dependencies": get_route_dependencies(), } diff --git a/stac_fastapi/tests/api/test_api.py b/stac_fastapi/tests/api/test_api.py index b16efd4c4..9387505b5 100644 --- a/stac_fastapi/tests/api/test_api.py +++ b/stac_fastapi/tests/api/test_api.py @@ -1516,33 +1516,6 @@ async def test_search_collection_limit_env_variable( assert int(limit) == len(resp_json["features"]) -@pytest.mark.asyncio -async def test_collection_items_limit_env_variable( - app_client, txn_client, load_test_data -): - limit = "5" - os.environ["STAC_ITEM_LIMIT"] = limit - - test_collection = load_test_data("test_collection.json") - test_collection_id = "test-collection-items-limit" - test_collection["id"] = test_collection_id - await create_collection(txn_client, test_collection) - - item = load_test_data("test_item.json") - item["collection"] = test_collection_id - - for i in range(10): - test_item = item.copy() - test_item["id"] = f"test-item-collection-{i}" - await create_item(txn_client, test_item) - - resp = await app_client.get(f"/collections/{test_collection_id}/items") - assert resp.status_code == 200 - resp_json = resp.json() - assert int(limit) == len(resp_json["features"]) - - -@pytest.mark.asyncio async def test_search_max_item_limit( app_client, load_test_data, txn_client, monkeypatch ): diff --git a/stac_fastapi/tests/api/test_api_item_collection.py b/stac_fastapi/tests/api/test_api_item_collection.py new file mode 100644 index 000000000..7c7b51b6f --- /dev/null +++ b/stac_fastapi/tests/api/test_api_item_collection.py @@ -0,0 +1,192 @@ +import json +import os +import uuid +from copy import deepcopy +from datetime import datetime, timedelta + +import pytest + +from ..conftest import create_collection, create_item + + +@pytest.mark.asyncio +async def test_item_collection_limit_env_variable( + app_client, txn_client, load_test_data +): + limit = "5" + os.environ["STAC_ITEM_LIMIT"] = limit + + test_collection = load_test_data("test_collection.json") + test_collection_id = "test-collection-items-limit" + test_collection["id"] = test_collection_id + await create_collection(txn_client, test_collection) + + item = load_test_data("test_item.json") + item["collection"] = test_collection_id + + for i in range(10): + test_item = item.copy() + test_item["id"] = f"test-item-collection-{i}" + await create_item(txn_client, test_item) + + resp = await app_client.get(f"/collections/{test_collection_id}/items") + assert resp.status_code == 200 + resp_json = resp.json() + assert int(limit) == len(resp_json["features"]) + + +@pytest.mark.asyncio +async def test_item_collection_sort_desc(app_client, txn_client, ctx): + """Verify GET /collections/{collectionId}/items honors descending sort on properties.datetime.""" + first_item = ctx.item + + # Create a second item in the same collection with an earlier datetime + second_item = dict(first_item) + second_item["id"] = "another-item-for-collection-sort-desc" + another_item_date = datetime.strptime( + first_item["properties"]["datetime"], "%Y-%m-%dT%H:%M:%SZ" + ) - timedelta(days=1) + second_item["properties"]["datetime"] = another_item_date.strftime( + "%Y-%m-%dT%H:%M:%SZ" + ) + + await create_item(txn_client, second_item) + + # Descending sort: the original (newer) item should come first + resp = await app_client.get( + f"/collections/{first_item['collection']}/items", + params=[("sortby", "-properties.datetime")], + ) + assert resp.status_code == 200 + resp_json = resp.json() + assert resp_json["features"][0]["id"] == first_item["id"] + assert resp_json["features"][1]["id"] == second_item["id"] + + +@pytest.mark.asyncio +async def test_item_collection_sort_asc(app_client, txn_client, ctx): + """Verify GET /collections/{collectionId}/items honors ascending sort on properties.datetime.""" + first_item = ctx.item + + # Create a second item in the same collection with an earlier datetime + second_item = dict(first_item) + second_item["id"] = "another-item-for-collection-sort-asc" + another_item_date = datetime.strptime( + first_item["properties"]["datetime"], "%Y-%m-%dT%H:%M:%SZ" + ) - timedelta(days=1) + second_item["properties"]["datetime"] = another_item_date.strftime( + "%Y-%m-%dT%H:%M:%SZ" + ) + + await create_item(txn_client, second_item) + + # Ascending sort: the older item should come first + resp = await app_client.get( + f"/collections/{first_item['collection']}/items", + params=[("sortby", "+properties.datetime")], + ) + assert resp.status_code == 200 + resp_json = resp.json() + assert resp_json["features"][0]["id"] == second_item["id"] + assert resp_json["features"][1]["id"] == first_item["id"] + + # Also verify bare field (no +) sorts ascending by default + resp = await app_client.get( + f"/collections/{first_item['collection']}/items", + params=[("sortby", "properties.datetime")], + ) + assert resp.status_code == 200 + resp_json = resp.json() + assert resp_json["features"][0]["id"] == second_item["id"] + assert resp_json["features"][1]["id"] == first_item["id"] + + +@pytest.mark.asyncio +async def test_item_collection_query(app_client, txn_client, ctx): + """Simple query parameter test on the Item Collection route. + + Creates an item with a unique property and ensures it can be retrieved + using the 'query' parameter on GET /collections/{collection_id}/items. + """ + unique_val = str(uuid.uuid4()) + test_item = deepcopy(ctx.item) + test_item["id"] = f"query-basic-{unique_val}" + # Add a property to filter on + test_item.setdefault("properties", {})["test_query_key"] = unique_val + + await create_item(txn_client, test_item) + + # Provide the query parameter as a JSON string without adding new imports + query_param = f'{{"test_query_key": {{"eq": "{unique_val}"}}}}' + + resp = await app_client.get( + f"/collections/{test_item['collection']}/items", + params=[("query", query_param)], + ) + assert resp.status_code == 200 + resp_json = resp.json() + ids = [f["id"] for f in resp_json["features"]] + assert test_item["id"] in ids + + +@pytest.mark.asyncio +async def test_item_collection_filter_by_id(app_client, ctx): + """Test filtering items by ID using the filter parameter.""" + # Get the test item and collection from the context + item = ctx.item + collection_id = item["collection"] + item_id = item["id"] + + # Create a filter to match the item by ID + filter_body = {"op": "=", "args": [{"property": "id"}, item_id]} + + # Make the request with the filter + params = [("filter", json.dumps(filter_body)), ("filter-lang", "cql2-json")] + + resp = await app_client.get( + f"/collections/{collection_id}/items", + params=params, + ) + + # Verify the response + assert resp.status_code == 200 + resp_json = resp.json() + + # Should find exactly one matching item + assert len(resp_json["features"]) == 1 + assert resp_json["features"][0]["id"] == item_id + assert resp_json["features"][0]["collection"] == collection_id + + +@pytest.mark.asyncio +async def test_item_collection_filter_by_nonexistent_id(app_client, ctx, txn_client): + """Test filtering with a non-existent ID returns no results.""" + # Get the test collection and item from context + collection_id = ctx.collection["id"] + item_id = ctx.item["id"] + + # First, verify the item exists + resp = await app_client.get(f"/collections/{collection_id}/items/{item_id}") + assert resp.status_code == 200 + + # Create a non-existent ID + non_existent_id = f"non-existent-{str(uuid.uuid4())}" + + # Create a filter with the non-existent ID using CQL2-JSON syntax + filter_body = {"op": "=", "args": [{"property": "id"}, non_existent_id]} + + # URL-encode the filter JSON + import urllib.parse + + encoded_filter = urllib.parse.quote(json.dumps(filter_body)) + + # Make the request with URL-encoded filter in the query string + url = f"/collections/{collection_id}/items?filter-lang=cql2-json&filter={encoded_filter}" + resp = await app_client.get(url) + + # Verify the response + assert resp.status_code == 200 + resp_json = resp.json() + assert ( + len(resp_json["features"]) == 0 + ), f"Expected no items with ID {non_existent_id}, but found {len(resp_json['features'])} matches"