Skip to content

Commit 196696c

Browse files
z-mrozujonhealy1
authored and
Yuri Zmytrakov
committed
Blocking bulk_items endpoint when ENABLE_DATETIME_INDEX_FILTERING (stac-utils#438)
**Description:** The `/collections/{collection_id}/bulk_items` endpoint does not work when `ENABLE_DATETIME_INDEX_FILTERING` is set to true, as only the async `/collections/{collection_id}/items` version was implemented while the sync `/collections/{collection_id}/bulk_items` wasn't. This PR blocks the /bulk_items endpoint when `ENABLE_DATETIME_INDEX_FILTERING` is set to true. **PR Checklist:** - [x] Code is formatted and linted (run `pre-commit run --all-files`) - [x] Tests pass (run `make test`) - [ ] Documentation has been updated to reflect changes, if applicable - [x] Changes are added to the changelog --------- Co-authored-by: Jonathan Healy <[email protected]>
1 parent ca26c0a commit 196696c

File tree

3 files changed

+98
-55
lines changed

3 files changed

+98
-55
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
1616
### Changed
1717

1818
- Changed assets serialization to prevent mapping explosion while allowing asset inforamtion to be indexed. [#341](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/341)
19+
- Blocked the /collections/{collection_id}/bulk_items endpoint when environmental variable ENABLE_DATETIME_INDEX_FILTERING is set to true. [#438](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/438)
1920

2021
### Fixed
2122

‎stac_fastapi/core/stac_fastapi/core/core.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -965,6 +965,13 @@ def bulk_item_insert(
965965
A string indicating the number of items successfully added.
966966
"""
967967
request = kwargs.get("request")
968+
969+
if os.getenv("ENABLE_DATETIME_INDEX_FILTERING"):
970+
raise HTTPException(
971+
status_code=400,
972+
detail="The /collections/{collection_id}/bulk_items endpoint is invalid when ENABLE_DATETIME_INDEX_FILTERING is set to true. Try using the /collections/{collection_id}/items endpoint.",
973+
)
974+
968975
if request:
969976
base_url = str(request.base_url)
970977
else:

‎stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py‎

Lines changed: 90 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
"""Database logic."""
2-
32
import asyncio
43
import logging
54
from base64 import urlsafe_b64decode, urlsafe_b64encode
@@ -16,7 +15,7 @@
1615

1716
from stac_fastapi.core.base_database_logic import BaseDatabaseLogic
1817
from stac_fastapi.core.serializers import CollectionSerializer, ItemSerializer
19-
from stac_fastapi.core.utilities import bbox2polygon, get_max_limit
18+
from stac_fastapi.core.utilities import bbox2polygon, get_bool_env, get_max_limit
2019
from stac_fastapi.extensions.core.transaction.request import (
2120
PartialCollection,
2221
PartialItem,
@@ -301,41 +300,22 @@ def apply_datetime_filter(
301300
if not datetime_search:
302301
return search, datetime_search
303302

304-
if "eq" in datetime_search:
305-
# For exact matches, include:
306-
# 1. Items with matching exact datetime
307-
# 2. Items with datetime:null where the time falls within their range
308-
should = [
309-
Q(
303+
# USE_DATETIME env var
304+
# True: Only search by datetime and ignore start/end datetime
305+
# False: Search by datetime, if null search by start/end datetime
306+
USE_DATETIME = get_bool_env("USE_DATETIME", default=False)
307+
308+
if USE_DATETIME:
309+
if "eq" in datetime_search:
310+
filter_query = Q(
310311
"bool",
311312
filter=[
312313
Q("exists", field="properties.datetime"),
313314
Q("term", **{"properties__datetime": datetime_search["eq"]}),
314315
],
315-
),
316-
Q(
317-
"bool",
318-
must_not=[Q("exists", field="properties.datetime")],
319-
filter=[
320-
Q("exists", field="properties.start_datetime"),
321-
Q("exists", field="properties.end_datetime"),
322-
Q(
323-
"range",
324-
properties__start_datetime={"lte": datetime_search["eq"]},
325-
),
326-
Q(
327-
"range",
328-
properties__end_datetime={"gte": datetime_search["eq"]},
329-
),
330-
],
331-
),
332-
]
333-
else:
334-
# For date ranges, include:
335-
# 1. Items with datetime in the range
336-
# 2. Items with datetime:null that overlap the search range
337-
should = [
338-
Q(
316+
)
317+
else:
318+
filter_query = Q(
339319
"bool",
340320
filter=[
341321
Q("exists", field="properties.datetime"),
@@ -347,29 +327,84 @@ def apply_datetime_filter(
347327
},
348328
),
349329
],
350-
),
351-
Q(
352-
"bool",
353-
must_not=[Q("exists", field="properties.datetime")],
354-
filter=[
355-
Q("exists", field="properties.start_datetime"),
356-
Q("exists", field="properties.end_datetime"),
357-
Q(
358-
"range",
359-
properties__start_datetime={"lte": datetime_search["lte"]},
360-
),
361-
Q(
362-
"range",
363-
properties__end_datetime={"gte": datetime_search["gte"]},
364-
),
365-
],
366-
),
367-
]
368-
369-
return (
370-
search.query(Q("bool", should=should, minimum_should_match=1)),
371-
datetime_search,
372-
)
330+
)
331+
return search.query(filter_query), datetime_search
332+
else:
333+
if "eq" in datetime_search:
334+
should = [
335+
Q(
336+
"bool",
337+
filter=[
338+
Q("exists", field="properties.datetime"),
339+
Q(
340+
"term",
341+
**{"properties__datetime": datetime_search["eq"]},
342+
),
343+
],
344+
),
345+
Q(
346+
"bool",
347+
must_not=[Q("exists", field="properties.datetime")],
348+
filter=[
349+
Q("exists", field="properties.start_datetime"),
350+
Q("exists", field="properties.end_datetime"),
351+
Q(
352+
"range",
353+
properties__start_datetime={
354+
"lte": datetime_search["eq"]
355+
},
356+
),
357+
Q(
358+
"range",
359+
properties__end_datetime={"gte": datetime_search["eq"]},
360+
),
361+
],
362+
),
363+
]
364+
else:
365+
# For date ranges, include:
366+
# 1. Items with datetime in the range
367+
# 2. Items with datetime:null that overlap the search range
368+
should = [
369+
Q(
370+
"bool",
371+
filter=[
372+
Q("exists", field="properties.datetime"),
373+
Q(
374+
"range",
375+
properties__datetime={
376+
"gte": datetime_search["gte"],
377+
"lte": datetime_search["lte"],
378+
},
379+
),
380+
],
381+
),
382+
Q(
383+
"bool",
384+
must_not=[Q("exists", field="properties.datetime")],
385+
filter=[
386+
Q("exists", field="properties.start_datetime"),
387+
Q("exists", field="properties.end_datetime"),
388+
Q(
389+
"range",
390+
properties__start_datetime={
391+
"lte": datetime_search["lte"]
392+
},
393+
),
394+
Q(
395+
"range",
396+
properties__end_datetime={
397+
"gte": datetime_search["gte"]
398+
},
399+
),
400+
],
401+
),
402+
]
403+
404+
return (
405+
search.query(Q("bool", should=should, minimum_should_match=1)),
406+
datetime_search,
407+
)
373408

374409
@staticmethod
375410
def apply_bbox_filter(search: Search, bbox: List):

0 commit comments

Comments
 (0)