From 7678a656cda1342ee4b0eeee1d5beff823deb680 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Mon, 10 Aug 2020 22:13:53 -0400 Subject: [PATCH 1/4] store db reader/writer dependencies in starlette app.state --- stac_api/app.py | 17 ++++++++--------- stac_api/utils/dependencies.py | 21 +++++++-------------- 2 files changed, 15 insertions(+), 23 deletions(-) diff --git a/stac_api/app.py b/stac_api/app.py index ebd940a5f..7a821c005 100644 --- a/stac_api/app.py +++ b/stac_api/app.py @@ -4,7 +4,6 @@ from . import settings from .resources import mgmt, collection, conformance, item -from .utils import dependencies app = FastAPI() @@ -18,18 +17,18 @@ @app.on_event("startup") async def on_startup(): """Create database engines and sessions on startup""" - dependencies.ENGINE_READER = create_engine(settings.SQLALCHEMY_DATABASE_READER) - dependencies.ENGINE_WRITER = create_engine(settings.SQLALCHEMY_DATABASE_WRITER) - dependencies.DB_READER = sessionmaker( - autocommit=False, autoflush=False, bind=dependencies.ENGINE_READER + app.state.ENGINE_READER = create_engine(settings.SQLALCHEMY_DATABASE_READER) + app.state.ENGINE_WRITER = create_engine(settings.SQLALCHEMY_DATABASE_WRITER) + app.state.DB_READER = sessionmaker( + autocommit=False, autoflush=False, bind=app.state.ENGINE_READER ) - dependencies.DB_WRITER = sessionmaker( - autocommit=False, autoflush=False, bind=dependencies.ENGINE_WRITER + app.state.DB_WRITER = sessionmaker( + autocommit=False, autoflush=False, bind=app.state.ENGINE_WRITER ) @app.on_event("shutdown") async def on_shutdown(): """Dispose of database engines and sessions on app shutdown""" - dependencies.ENGINE_READER.dispose() - dependencies.ENGINE_WRITER.dispose() + app.state.ENGINE_READER.dispose() + app.state.ENGINE_WRITER.dispose() diff --git a/stac_api/utils/dependencies.py b/stac_api/utils/dependencies.py index 892bff856..301eb3cdf 100644 --- a/stac_api/utils/dependencies.py +++ b/stac_api/utils/dependencies.py @@ -1,17 +1,10 @@ from dataclasses import dataclass -from typing import Callable, List, Optional +from typing import Callable, List -from sqlalchemy.engine import Engine from sqlalchemy.orm import Session from starlette.requests import Request -ENGINE_READER: Optional[Engine] = None -ENGINE_WRITER: Optional[Engine] = None -DB_READER: Optional[Session] = None -DB_WRITER: Optional[Session] = None - - @dataclass class DatabaseConnectionError(Exception): message: str @@ -32,27 +25,27 @@ def _parse(request: Request): return _parse -def database_reader_factory() -> Session: +def database_reader_factory(request: Request) -> Session: """Instantiate the database reader session""" try: - if not DB_READER: + if not request.app.state.DB_READER: raise DatabaseConnectionError( message="Database engine has not been created" ) - db = DB_READER() + db = request.app.state.DB_READER() yield db finally: db.close() -def database_writer_factory() -> Session: +def database_writer_factory(request: Request) -> Session: """Instantiate the database writer session""" try: - if not DB_WRITER: + if not request.app.state.DB_WRITER: raise DatabaseConnectionError( message="Database engine has not been created" ) - db = DB_WRITER() + db = request.app.state.DB_WRITER() yield db finally: db.close() From b00cb2e89b8400557b09696a8668882311754093 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Mon, 10 Aug 2020 22:40:06 -0400 Subject: [PATCH 2/4] remove useless if --- stac_api/utils/dependencies.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/stac_api/utils/dependencies.py b/stac_api/utils/dependencies.py index 301eb3cdf..bb7e0823f 100644 --- a/stac_api/utils/dependencies.py +++ b/stac_api/utils/dependencies.py @@ -28,10 +28,6 @@ def _parse(request: Request): def database_reader_factory(request: Request) -> Session: """Instantiate the database reader session""" try: - if not request.app.state.DB_READER: - raise DatabaseConnectionError( - message="Database engine has not been created" - ) db = request.app.state.DB_READER() yield db finally: @@ -41,10 +37,6 @@ def database_reader_factory(request: Request) -> Session: def database_writer_factory(request: Request) -> Session: """Instantiate the database writer session""" try: - if not request.app.state.DB_WRITER: - raise DatabaseConnectionError( - message="Database engine has not been created" - ) db = request.app.state.DB_WRITER() yield db finally: From 3d3de4203ae1bc6c2cd928a18a26e79f7a51d2c3 Mon Sep 17 00:00:00 2001 From: Jeff Albrecht Date: Tue, 11 Aug 2020 19:41:11 -0500 Subject: [PATCH 3/4] shutdown reader on app shutdown, remove reader/writer deps --- stac_api/app.py | 6 ++++-- stac_api/clients/collection_crud.py | 11 +++++------ stac_api/clients/item_crud.py | 10 ++++------ stac_api/clients/tokens.py | 13 ++++--------- stac_api/utils/dependencies.py | 19 ------------------- 5 files changed, 17 insertions(+), 42 deletions(-) diff --git a/stac_api/app.py b/stac_api/app.py index 7a821c005..d2a3fb726 100644 --- a/stac_api/app.py +++ b/stac_api/app.py @@ -21,10 +21,10 @@ async def on_startup(): app.state.ENGINE_WRITER = create_engine(settings.SQLALCHEMY_DATABASE_WRITER) app.state.DB_READER = sessionmaker( autocommit=False, autoflush=False, bind=app.state.ENGINE_READER - ) + )() app.state.DB_WRITER = sessionmaker( autocommit=False, autoflush=False, bind=app.state.ENGINE_WRITER - ) + )() @app.on_event("shutdown") @@ -32,3 +32,5 @@ async def on_shutdown(): """Dispose of database engines and sessions on app shutdown""" app.state.ENGINE_READER.dispose() app.state.ENGINE_WRITER.dispose() + app.state.DB_READER.close() + app.state.DB_WRITER.close() diff --git a/stac_api/clients/collection_crud.py b/stac_api/clients/collection_crud.py index fa1456083..9e31d9cd4 100644 --- a/stac_api/clients/collection_crud.py +++ b/stac_api/clients/collection_crud.py @@ -3,14 +3,14 @@ from typing import List, Tuple from fastapi import Depends -from sqlalchemy.orm import Session from sqlakeyset import get_page, Page +from starlette.requests import Request from .base_crud import BaseCrudClient from .tokens import PaginationTokenClient, pagination_token_client_factory from .. import errors from ..models import database -from ..utils.dependencies import database_reader_factory, database_writer_factory + logger = logging.getLogger(__name__) @@ -63,13 +63,12 @@ def get_item_collection( def collection_crud_client_factory( - reader_session: Session = Depends(database_reader_factory), - writer_session: Session = Depends(database_writer_factory), + request: Request, pagination_client: PaginationTokenClient = Depends(pagination_token_client_factory), ) -> CollectionCrudClient: return CollectionCrudClient( - reader_session=reader_session, - writer_session=writer_session, + reader_session=request.app.state.DB_READER, + writer_session=request.app.state.DB_WRITER, table=database.Collection, pagination_client=pagination_client, ) diff --git a/stac_api/clients/item_crud.py b/stac_api/clients/item_crud.py index 385d8dac4..8914bdcd4 100644 --- a/stac_api/clients/item_crud.py +++ b/stac_api/clients/item_crud.py @@ -5,15 +5,14 @@ from fastapi import Depends import geoalchemy2 as ga import sqlalchemy as sa -from sqlalchemy.orm import Session from sqlakeyset import get_page, Page +from starlette.requests import Request from .base_crud import BaseCrudClient from .collection_crud import CollectionCrudClient, collection_crud_client_factory from .tokens import PaginationTokenClient, pagination_token_client_factory from ..errors import DatabaseError from ..models import database, schemas -from ..utils.dependencies import database_writer_factory, database_reader_factory logger = logging.getLogger(__name__) @@ -131,14 +130,13 @@ def stac_search(self, search_request: schemas.STACSearch) -> Tuple[Page, int]: def item_crud_client_factory( - reader_session: Session = Depends(database_reader_factory), - writer_session: Session = Depends(database_writer_factory), + request: Request, collection_crud: CollectionCrudClient = Depends(collection_crud_client_factory), pagination_client: PaginationTokenClient = Depends(pagination_token_client_factory), ) -> ItemCrudClient: return ItemCrudClient( - reader_session=reader_session, - writer_session=writer_session, + reader_session=request.app.state.DB_READER, + writer_session=request.app.state.DB_WRITER, collection_crud=collection_crud, table=database.Item, pagination_client=pagination_client, diff --git a/stac_api/clients/tokens.py b/stac_api/clients/tokens.py index 45481488a..11345854f 100644 --- a/stac_api/clients/tokens.py +++ b/stac_api/clients/tokens.py @@ -2,14 +2,12 @@ from dataclasses import dataclass import os -from fastapi import Depends -from sqlalchemy.orm import Session +from starlette.requests import Request from .base_crud import BaseCrudClient from ..models import database from ..errors import DatabaseError -from ..utils.dependencies import database_reader_factory, database_writer_factory @dataclass @@ -36,12 +34,9 @@ def get(self, token_id: str) -> str: return row.keyset -def pagination_token_client_factory( - reader_session: Session = Depends(database_reader_factory), - writer_session: Session = Depends(database_writer_factory), -) -> PaginationTokenClient: +def pagination_token_client_factory(request: Request) -> PaginationTokenClient: return PaginationTokenClient( - reader_session=reader_session, - writer_session=writer_session, + reader_session=request.app.state.DB_READER, + writer_session=request.app.state.DB_WRITER, table=database.PaginationToken, ) diff --git a/stac_api/utils/dependencies.py b/stac_api/utils/dependencies.py index bb7e0823f..5abf0b587 100644 --- a/stac_api/utils/dependencies.py +++ b/stac_api/utils/dependencies.py @@ -1,7 +1,6 @@ from dataclasses import dataclass from typing import Callable, List -from sqlalchemy.orm import Session from starlette.requests import Request @@ -23,21 +22,3 @@ def _parse(request: Request): return param.split(",") if param else param return _parse - - -def database_reader_factory(request: Request) -> Session: - """Instantiate the database reader session""" - try: - db = request.app.state.DB_READER() - yield db - finally: - db.close() - - -def database_writer_factory(request: Request) -> Session: - """Instantiate the database writer session""" - try: - db = request.app.state.DB_WRITER() - yield db - finally: - db.close() From 9565f3c592076f6380a8d696bcd95ad2ccd3e324 Mon Sep 17 00:00:00 2001 From: Jeff Albrecht Date: Tue, 11 Aug 2020 19:48:11 -0500 Subject: [PATCH 4/4] Revert "shutdown reader on app shutdown, remove reader/writer deps" This reverts commit 3d3de4203ae1bc6c2cd928a18a26e79f7a51d2c3. --- stac_api/app.py | 6 ++---- stac_api/clients/collection_crud.py | 11 ++++++----- stac_api/clients/item_crud.py | 10 ++++++---- stac_api/clients/tokens.py | 13 +++++++++---- stac_api/utils/dependencies.py | 19 +++++++++++++++++++ 5 files changed, 42 insertions(+), 17 deletions(-) diff --git a/stac_api/app.py b/stac_api/app.py index d2a3fb726..7a821c005 100644 --- a/stac_api/app.py +++ b/stac_api/app.py @@ -21,10 +21,10 @@ async def on_startup(): app.state.ENGINE_WRITER = create_engine(settings.SQLALCHEMY_DATABASE_WRITER) app.state.DB_READER = sessionmaker( autocommit=False, autoflush=False, bind=app.state.ENGINE_READER - )() + ) app.state.DB_WRITER = sessionmaker( autocommit=False, autoflush=False, bind=app.state.ENGINE_WRITER - )() + ) @app.on_event("shutdown") @@ -32,5 +32,3 @@ async def on_shutdown(): """Dispose of database engines and sessions on app shutdown""" app.state.ENGINE_READER.dispose() app.state.ENGINE_WRITER.dispose() - app.state.DB_READER.close() - app.state.DB_WRITER.close() diff --git a/stac_api/clients/collection_crud.py b/stac_api/clients/collection_crud.py index 9e31d9cd4..fa1456083 100644 --- a/stac_api/clients/collection_crud.py +++ b/stac_api/clients/collection_crud.py @@ -3,14 +3,14 @@ from typing import List, Tuple from fastapi import Depends +from sqlalchemy.orm import Session from sqlakeyset import get_page, Page -from starlette.requests import Request from .base_crud import BaseCrudClient from .tokens import PaginationTokenClient, pagination_token_client_factory from .. import errors from ..models import database - +from ..utils.dependencies import database_reader_factory, database_writer_factory logger = logging.getLogger(__name__) @@ -63,12 +63,13 @@ def get_item_collection( def collection_crud_client_factory( - request: Request, + reader_session: Session = Depends(database_reader_factory), + writer_session: Session = Depends(database_writer_factory), pagination_client: PaginationTokenClient = Depends(pagination_token_client_factory), ) -> CollectionCrudClient: return CollectionCrudClient( - reader_session=request.app.state.DB_READER, - writer_session=request.app.state.DB_WRITER, + reader_session=reader_session, + writer_session=writer_session, table=database.Collection, pagination_client=pagination_client, ) diff --git a/stac_api/clients/item_crud.py b/stac_api/clients/item_crud.py index 8914bdcd4..385d8dac4 100644 --- a/stac_api/clients/item_crud.py +++ b/stac_api/clients/item_crud.py @@ -5,14 +5,15 @@ from fastapi import Depends import geoalchemy2 as ga import sqlalchemy as sa +from sqlalchemy.orm import Session from sqlakeyset import get_page, Page -from starlette.requests import Request from .base_crud import BaseCrudClient from .collection_crud import CollectionCrudClient, collection_crud_client_factory from .tokens import PaginationTokenClient, pagination_token_client_factory from ..errors import DatabaseError from ..models import database, schemas +from ..utils.dependencies import database_writer_factory, database_reader_factory logger = logging.getLogger(__name__) @@ -130,13 +131,14 @@ def stac_search(self, search_request: schemas.STACSearch) -> Tuple[Page, int]: def item_crud_client_factory( - request: Request, + reader_session: Session = Depends(database_reader_factory), + writer_session: Session = Depends(database_writer_factory), collection_crud: CollectionCrudClient = Depends(collection_crud_client_factory), pagination_client: PaginationTokenClient = Depends(pagination_token_client_factory), ) -> ItemCrudClient: return ItemCrudClient( - reader_session=request.app.state.DB_READER, - writer_session=request.app.state.DB_WRITER, + reader_session=reader_session, + writer_session=writer_session, collection_crud=collection_crud, table=database.Item, pagination_client=pagination_client, diff --git a/stac_api/clients/tokens.py b/stac_api/clients/tokens.py index 11345854f..45481488a 100644 --- a/stac_api/clients/tokens.py +++ b/stac_api/clients/tokens.py @@ -2,12 +2,14 @@ from dataclasses import dataclass import os -from starlette.requests import Request +from fastapi import Depends +from sqlalchemy.orm import Session from .base_crud import BaseCrudClient from ..models import database from ..errors import DatabaseError +from ..utils.dependencies import database_reader_factory, database_writer_factory @dataclass @@ -34,9 +36,12 @@ def get(self, token_id: str) -> str: return row.keyset -def pagination_token_client_factory(request: Request) -> PaginationTokenClient: +def pagination_token_client_factory( + reader_session: Session = Depends(database_reader_factory), + writer_session: Session = Depends(database_writer_factory), +) -> PaginationTokenClient: return PaginationTokenClient( - reader_session=request.app.state.DB_READER, - writer_session=request.app.state.DB_WRITER, + reader_session=reader_session, + writer_session=writer_session, table=database.PaginationToken, ) diff --git a/stac_api/utils/dependencies.py b/stac_api/utils/dependencies.py index 5abf0b587..bb7e0823f 100644 --- a/stac_api/utils/dependencies.py +++ b/stac_api/utils/dependencies.py @@ -1,6 +1,7 @@ from dataclasses import dataclass from typing import Callable, List +from sqlalchemy.orm import Session from starlette.requests import Request @@ -22,3 +23,21 @@ def _parse(request: Request): return param.split(",") if param else param return _parse + + +def database_reader_factory(request: Request) -> Session: + """Instantiate the database reader session""" + try: + db = request.app.state.DB_READER() + yield db + finally: + db.close() + + +def database_writer_factory(request: Request) -> Session: + """Instantiate the database writer session""" + try: + db = request.app.state.DB_WRITER() + yield db + finally: + db.close()