Skip to content

Commit 0178a4f

Browse files
update pagination for collection-search (#155)
* add failing test * handle collection paging differently (#156) * handle collection paging differently * test next link * add OffsetPaginationExtension * uncomment test * update to pgstac 0.9.2 * update prev logic * only test 0.9.0 * update pypstac version * add back 0.8 support but allow skip tests * skip for 0.8 * remove warnings * fallback to all_collections when `CollectionSearchExtension` is not enabled (#179) * fallback to all_collections when `CollectionSearchExtension` is not enabled * test all_collection fallback * add offset=0 * Update tests/conftest.py Co-authored-by: Henry Rodman <[email protected]> --------- Co-authored-by: Henry Rodman <[email protected]>
1 parent 7135059 commit 0178a4f

File tree

8 files changed

+361
-37
lines changed

8 files changed

+361
-37
lines changed

.github/workflows/cicd.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ jobs:
1515
- {python: '3.12', pypgstac: '0.9.*'}
1616
- {python: '3.12', pypgstac: '0.8.*'}
1717
- {python: '3.11', pypgstac: '0.8.*'}
18+
- {python: '3.10', pypgstac: '0.8.*'}
1819
- {python: '3.9', pypgstac: '0.8.*'}
1920
- {python: '3.8', pypgstac: '0.8.*'}
2021

docker-compose.yml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@ services:
3535
build:
3636
context: .
3737
dockerfile: Dockerfile.tests
38+
volumes:
39+
- .:/app
3840
environment:
3941
- ENVIRONMENT=local
4042
- DB_MIN_CONN_SIZE=1
@@ -44,7 +46,7 @@ services:
4446

4547
database:
4648
container_name: stac-db
47-
image: ghcr.io/stac-utils/pgstac:v0.9.1
49+
image: ghcr.io/stac-utils/pgstac:v0.9.2
4850
environment:
4951
- POSTGRES_USER=username
5052
- POSTGRES_PASSWORD=password

setup.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,9 @@
1010
"orjson",
1111
"pydantic",
1212
"stac_pydantic==3.1.*",
13-
"stac-fastapi.api~=3.0.2",
14-
"stac-fastapi.extensions~=3.0.2",
15-
"stac-fastapi.types~=3.0.2",
13+
"stac-fastapi.api~=3.0.3",
14+
"stac-fastapi.extensions~=3.0.3",
15+
"stac-fastapi.types~=3.0.3",
1616
"asyncpg",
1717
"buildpg",
1818
"brotli_asgi",

stac_fastapi/pgstac/app.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
from stac_fastapi.extensions.core import (
2222
FieldsExtension,
2323
FilterExtension,
24+
OffsetPaginationExtension,
2425
SortExtension,
2526
TokenPaginationExtension,
2627
TransactionExtension,
@@ -58,6 +59,7 @@
5859
"sort": SortExtension(),
5960
"fields": FieldsExtension(),
6061
"filter": FilterExtension(client=FiltersClient()),
62+
"pagination": OffsetPaginationExtension(),
6163
}
6264

6365
enabled_extensions = (

stac_fastapi/pgstac/core.py

Lines changed: 47 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
from stac_fastapi.pgstac.config import Settings
2626
from stac_fastapi.pgstac.models.links import (
2727
CollectionLinks,
28+
CollectionSearchPagingLinks,
2829
ItemCollectionLinks,
2930
ItemLinks,
3031
PagingLinks,
@@ -46,8 +47,8 @@ async def all_collections( # noqa: C901
4647
bbox: Optional[BBox] = None,
4748
datetime: Optional[DateTimeType] = None,
4849
limit: Optional[int] = None,
50+
offset: Optional[int] = None,
4951
query: Optional[str] = None,
50-
token: Optional[str] = None,
5152
fields: Optional[List[str]] = None,
5253
sortby: Optional[str] = None,
5354
filter: Optional[str] = None,
@@ -64,38 +65,51 @@ async def all_collections( # noqa: C901
6465
"""
6566
base_url = get_base_url(request)
6667

67-
# Parse request parameters
68-
base_args = {
69-
"bbox": bbox,
70-
"limit": limit,
71-
"token": token,
72-
"query": orjson.loads(unquote_plus(query)) if query else query,
73-
}
74-
75-
clean_args = clean_search_args(
76-
base_args=base_args,
77-
datetime=datetime,
78-
fields=fields,
79-
sortby=sortby,
80-
filter_query=filter,
81-
filter_lang=filter_lang,
82-
)
83-
84-
async with request.app.state.get_connection(request, "r") as conn:
85-
q, p = render(
86-
"""
87-
SELECT * FROM collection_search(:req::text::jsonb);
88-
""",
89-
req=json.dumps(clean_args),
68+
next_link: Optional[Dict[str, Any]] = None
69+
prev_link: Optional[Dict[str, Any]] = None
70+
collections_result: Collections
71+
72+
if self.extension_is_enabled("CollectionSearchExtension"):
73+
base_args = {
74+
"bbox": bbox,
75+
"limit": limit,
76+
"offset": offset,
77+
"query": orjson.loads(unquote_plus(query)) if query else query,
78+
}
79+
80+
clean_args = clean_search_args(
81+
base_args=base_args,
82+
datetime=datetime,
83+
fields=fields,
84+
sortby=sortby,
85+
filter_query=filter,
86+
filter_lang=filter_lang,
9087
)
91-
collections_result: Collections = await conn.fetchval(q, *p)
9288

93-
next: Optional[str] = None
94-
prev: Optional[str] = None
89+
async with request.app.state.get_connection(request, "r") as conn:
90+
q, p = render(
91+
"""
92+
SELECT * FROM collection_search(:req::text::jsonb);
93+
""",
94+
req=json.dumps(clean_args),
95+
)
96+
collections_result = await conn.fetchval(q, *p)
9597

96-
if links := collections_result.get("links"):
97-
next = collections_result["links"].pop("next")
98-
prev = collections_result["links"].pop("prev")
98+
if links := collections_result.get("links"):
99+
for link in links:
100+
if link["rel"] == "next":
101+
next_link = link
102+
elif link["rel"] == "prev":
103+
prev_link = link
104+
105+
else:
106+
async with request.app.state.get_connection(request, "r") as conn:
107+
cols = await conn.fetchval(
108+
"""
109+
SELECT * FROM all_collections();
110+
"""
111+
)
112+
collections_result = {"collections": cols, "links": []}
99113

100114
linked_collections: List[Collection] = []
101115
collections = collections_result["collections"]
@@ -120,10 +134,10 @@ async def all_collections( # noqa: C901
120134

121135
linked_collections.append(coll)
122136

123-
links = await PagingLinks(
137+
links = await CollectionSearchPagingLinks(
124138
request=request,
125-
next=next,
126-
prev=prev,
139+
next=next_link,
140+
prev=prev_link,
127141
).get_links()
128142

129143
return Collections(

stac_fastapi/pgstac/models/links.py

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,55 @@ def link_prev(self) -> Optional[Dict[str, Any]]:
173173
return None
174174

175175

176+
@attr.s
177+
class CollectionSearchPagingLinks(BaseLinks):
178+
next: Optional[Dict[str, Any]] = attr.ib(kw_only=True, default=None)
179+
prev: Optional[Dict[str, Any]] = attr.ib(kw_only=True, default=None)
180+
181+
def link_next(self) -> Optional[Dict[str, Any]]:
182+
"""Create link for next page."""
183+
if self.next is not None:
184+
method = self.request.method
185+
if method == "GET":
186+
# if offset is equal to default value (0), drop it
187+
if self.next["body"].get("offset", -1) == 0:
188+
_ = self.next["body"].pop("offset")
189+
190+
href = merge_params(self.url, self.next["body"])
191+
192+
# if next link is equal to this link, skip it
193+
if href == self.url:
194+
return None
195+
196+
return {
197+
"rel": Relations.next.value,
198+
"type": MimeTypes.geojson.value,
199+
"method": method,
200+
"href": href,
201+
}
202+
203+
return None
204+
205+
def link_prev(self):
206+
if self.prev is not None:
207+
method = self.request.method
208+
if method == "GET":
209+
href = merge_params(self.url, self.prev["body"])
210+
211+
# if prev link is equal to this link, skip it
212+
if href == self.url:
213+
return None
214+
215+
return {
216+
"rel": Relations.previous.value,
217+
"type": MimeTypes.geojson.value,
218+
"method": method,
219+
"href": href,
220+
}
221+
222+
return None
223+
224+
176225
@attr.s
177226
class CollectionLinksBase(BaseLinks):
178227
"""Create inferred links specific to collections."""

tests/conftest.py

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
from fastapi import APIRouter
1313
from fastapi.responses import ORJSONResponse
1414
from httpx import ASGITransport, AsyncClient
15+
from pypgstac import __version__ as pgstac_version
1516
from pypgstac.db import PgstacDB
1617
from pypgstac.migrate import Migrate
1718
from pytest_postgresql.janitor import DatabaseJanitor
@@ -26,6 +27,7 @@
2627
CollectionSearchExtension,
2728
FieldsExtension,
2829
FilterExtension,
30+
OffsetPaginationExtension,
2931
SortExtension,
3032
TokenPaginationExtension,
3133
TransactionExtension,
@@ -47,6 +49,12 @@
4749
logger = logging.getLogger(__name__)
4850

4951

52+
requires_pgstac_0_9_2 = pytest.mark.skipif(
53+
tuple(map(int, pgstac_version.split("."))) < (0, 9, 2),
54+
reason="PgSTAC>=0.9.2 required",
55+
)
56+
57+
5058
@pytest.fixture(scope="session")
5159
def event_loop():
5260
return asyncio.get_event_loop()
@@ -140,6 +148,7 @@ def api_client(request, database):
140148
SortExtension(),
141149
FieldsExtension(),
142150
FilterExtension(client=FiltersClient()),
151+
OffsetPaginationExtension(),
143152
]
144153
collection_search_extension = CollectionSearchExtension.from_extensions(
145154
collection_extensions
@@ -259,3 +268,48 @@ async def load_test2_item(app_client, load_test_data, load_test2_collection):
259268
)
260269
assert resp.status_code == 201
261270
return Item.model_validate(resp.json())
271+
272+
273+
@pytest.fixture(
274+
scope="session",
275+
)
276+
def api_client_no_ext(database):
277+
api_settings = Settings(
278+
postgres_user=database.user,
279+
postgres_pass=database.password,
280+
postgres_host_reader=database.host,
281+
postgres_host_writer=database.host,
282+
postgres_port=database.port,
283+
postgres_dbname=database.dbname,
284+
testing=True,
285+
)
286+
return StacApi(
287+
settings=api_settings,
288+
extensions=[
289+
TransactionExtension(client=TransactionsClient(), settings=api_settings)
290+
],
291+
client=CoreCrudClient(),
292+
)
293+
294+
295+
@pytest.fixture(scope="function")
296+
async def app_no_ext(api_client_no_ext):
297+
logger.info("Creating app Fixture")
298+
time.time()
299+
app = api_client_no_ext.app
300+
await connect_to_db(app)
301+
302+
yield app
303+
304+
await close_db_connection(app)
305+
306+
logger.info("Closed Pools.")
307+
308+
309+
@pytest.fixture(scope="function")
310+
async def app_client_no_ext(app_no_ext):
311+
logger.info("creating app_client")
312+
async with AsyncClient(
313+
transport=ASGITransport(app=app_no_ext), base_url="http://test"
314+
) as c:
315+
yield c

0 commit comments

Comments
 (0)