From b2b6f4238adae0110532549ded9a81d2104254bd Mon Sep 17 00:00:00 2001 From: Rouven Bauer Date: Wed, 3 Aug 2022 16:51:41 +0200 Subject: [PATCH 01/26] Initial draft of bookmark manager --- neo4j/_async/bookmark_manager.py | 86 +++++++++++++++++++++++++++++++ neo4j/_async/driver.py | 45 ++++++++++++++++ neo4j/_async/work/session.py | 21 ++++---- neo4j/_async/work/workspace.py | 39 +++++++++++++- neo4j/_conf.py | 4 ++ neo4j/_sync/bookmark_manager.py | 86 +++++++++++++++++++++++++++++++ neo4j/_sync/driver.py | 44 ++++++++++++++++ neo4j/_sync/work/session.py | 21 ++++---- neo4j/_sync/work/workspace.py | 39 +++++++++++++- neo4j/api.py | 88 ++++++++++++++++++++++++++++++++ 10 files changed, 449 insertions(+), 24 deletions(-) create mode 100644 neo4j/_async/bookmark_manager.py create mode 100644 neo4j/_sync/bookmark_manager.py diff --git a/neo4j/_async/bookmark_manager.py b/neo4j/_async/bookmark_manager.py new file mode 100644 index 000000000..ee0545bd7 --- /dev/null +++ b/neo4j/_async/bookmark_manager.py @@ -0,0 +1,86 @@ +# Copyright (c) "Neo4j" +# Neo4j Sweden AB [https://neo4j.com] +# +# This file is part of Neo4j. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import annotations + +import typing as t +from collections import defaultdict + +from .._async_compat.concurrency import ( + AsyncCooperativeLock, + AsyncLock, +) +from .._async_compat.util import AsyncUtil +from ..api import AsyncBookmarkManager + + +class AsyncNeo4jBookmarkManager(AsyncBookmarkManager): + def __init__(self, initial_bookmarks=None, bookmark_supplier=None, + notify_bookmarks=None): + super().__init__() + self._initial_bookmarks = initial_bookmarks + self._bookmark_supplier = bookmark_supplier + self._notify_bookmarks = notify_bookmarks + self._bookmarks = defaultdict(set) + if bookmark_supplier or notify_bookmarks: + self._lock = AsyncLock() + else: + self._lock = AsyncCooperativeLock() + + async def update_bookmarks( + self, database: str, previous_bookmarks: t.Iterable[str], + new_bookmarks: t.Iterable[str] + ) -> None: + with self._lock: + new_bms = set(new_bookmarks) + if not new_bms: + return + prev_bms = set(previous_bookmarks) + curr_bms = self._bookmarks[database] + curr_bms.difference_update(prev_bms) + curr_bms.update(new_bms) + + if self._notify_bookmarks: + await AsyncUtil.callback( + self._notify_bookmarks, database, tuple(curr_bms) + ) + + async def _get_bookmarks(self, database: str) -> t.Set[str]: + bms = self._bookmarks[database] + if self._bookmark_supplier: + extra_bms = await AsyncUtil.callback( + self._bookmark_supplier, database + ) + if extra_bms is not None: + bms &= set(extra_bms) + return bms + + async def get_bookmarks(self, database: str) -> t.Set[str]: + with self._lock: + return await self._get_bookmarks(database) + + async def get_all_bookmarks( + self, must_included_databases: t.Iterable[str] + ) -> t.Set[str]: + with self._lock: + bms = set() + databases = (set(must_included_databases) + | set(self._bookmarks.keys())) + for database in databases: + bms.update(await self._get_bookmarks(database)) + return bms diff --git a/neo4j/_async/driver.py b/neo4j/_async/driver.py index 6107f17d6..0b21d4d02 100644 --- a/neo4j/_async/driver.py +++ b/neo4j/_async/driver.py @@ -43,7 +43,9 @@ ) from ..addressing import Address from ..api import ( + AsyncBookmarkManager, Auth, + BookmarkManager, Bookmarks, DRIVER_BOLT, DRIVER_NEO4J, @@ -62,9 +64,14 @@ URI_SCHEME_NEO4J_SECURE, URI_SCHEME_NEO4J_SELF_SIGNED_CERTIFICATE, ) +from .bookmark_manager import AsyncNeo4jBookmarkManager from .work import AsyncSession +_T_BmSupplier = t.Callable[[str], t.Union[Bookmarks, t.Awaitable[Bookmarks]]] +_T_NotifyBm = t.Callable[[str, Bookmarks], t.Union[None, t.Awaitable[None]]] + + class AsyncGraphDatabase: """Accessor for :class:`neo4j.Driver` construction. """ @@ -94,6 +101,8 @@ def driver( ssl_context: ssl.SSLContext = ..., user_agent: str = ..., keep_alive: bool = ..., + bookmark_manager: t.Union[AsyncBookmarkManager, + BookmarkManager, None] = ..., # undocumented/unsupported options # they may be change or removed any time without prior notice @@ -208,6 +217,36 @@ def driver(cls, uri, *, auth=None, **config) -> AsyncDriver: return cls.neo4j_driver(parsed.netloc, auth=auth, routing_context=routing_context, **config) + @classmethod + def bookmark_manager( + cls, initial_bookmarks: Bookmarks = None, + bookmark_supplier: _T_BmSupplier = None, + notify_bookmarks: _T_NotifyBm = None + ) -> AsyncBookmarkManager: + """Create a default :class:`AsyncBookmarkManager`. + + :param initial_bookmarks: + The initial set of bookmarks. The default bookmark manager will + seed the set of bookmarks for each database with this value. + :param bookmark_supplier: + Function which will be called every time the default bookmark + manager's method :meth:`.AsyncBookmarkManager.get_bookmarks` + gets called. The result of ``bookmark_supplier`` will be + concatenated with the internal set of bookmarks and used to + configure the session in creation. + :param notify_bookmarks: + Function which will be called whenever the set of bookmarks + handled by the bookmark manager gets updated with the new + internal bookmark set. + + :returns: A default implementation of :class:`AsyncBookmarkManager`. + """ + return AsyncNeo4jBookmarkManager( + initial_bookmarks=initial_bookmarks, + bookmark_supplier=bookmark_supplier, + notify_bookmarks=notify_bookmarks + ) + @classmethod def bolt_driver(cls, target, *, auth=None, **config): """ Create a driver for direct Bolt server access that uses @@ -353,6 +392,8 @@ def session( initial_retry_delay: float = ..., retry_delay_multiplier: float = ..., retry_delay_jitter_factor: float = ..., + bookmark_manager: t.Union[AsyncBookmarkManager, + BookmarkManager, None] = ..., ) -> AsyncSession: ... @@ -394,6 +435,8 @@ async def verify_connectivity( initial_retry_delay: float = ..., retry_delay_multiplier: float = ..., retry_delay_jitter_factor: float = ..., + bookmark_manager: t.Union[AsyncBookmarkManager, + BookmarkManager, None] = ..., ) -> None: ... @@ -456,6 +499,8 @@ async def get_server_info( initial_retry_delay: float = ..., retry_delay_multiplier: float = ..., retry_delay_jitter_factor: float = ..., + bookmark_manager: t.Union[AsyncBookmarkManager, + BookmarkManager, None] = ..., ) -> ServerInfo: ... diff --git a/neo4j/_async/work/session.py b/neo4j/_async/work/session.py index 283aa23cd..06301a715 100644 --- a/neo4j/_async/work/session.py +++ b/neo4j/_async/work/session.py @@ -100,9 +100,12 @@ class AsyncSession(AsyncWorkspace): _state_failed = False def __init__(self, pool, session_config): - super().__init__(pool, session_config) assert isinstance(session_config, SessionConfig) - self._bookmarks = self._prepare_bookmarks(session_config.bookmarks) + super().__init__(pool, session_config) + if session_config.bookmarks is not None: + self._bookmarks = self._prepare_bookmarks(session_config.bookmarks) + else: + self._bookmark_manager = session_config.bookmark_manager async def __aenter__(self) -> AsyncSession: return self @@ -147,10 +150,6 @@ async def _disconnect(self, sync=False): self._handle_cancellation(message="_disconnect") raise - def _collect_bookmark(self, bookmark): - if bookmark: - self._bookmarks = bookmark, - def _handle_cancellation(self, message="General"): self._transaction = None self._auto_result = None @@ -165,7 +164,7 @@ def _handle_cancellation(self, message="General"): async def _result_closed(self): if self._auto_result: - self._collect_bookmark(self._auto_result._bookmark) + await self._update_bookmark(self._auto_result._bookmark) self._auto_result = None await self._disconnect() @@ -196,7 +195,7 @@ async def close(self) -> None: if self._state_failed is False: try: await self._auto_result.consume() - self._collect_bookmark(self._auto_result._bookmark) + await self._update_bookmark(self._auto_result._bookmark) except Exception as error: # TODO: Investigate potential non graceful close states self._auto_result = None @@ -336,7 +335,7 @@ async def last_bookmark(self) -> t.Optional[str]: await self._auto_result.consume() if self._transaction and self._transaction._closed: - self._collect_bookmark(self._transaction._bookmark) + await self._update_bookmark(self._transaction._bookmark) self._transaction = None if self._bookmarks: @@ -377,14 +376,14 @@ async def last_bookmarks(self) -> Bookmarks: await self._auto_result.consume() if self._transaction and self._transaction._closed(): - self._collect_bookmark(self._transaction._bookmark) + await self._update_bookmark(self._transaction._bookmark) self._transaction = None return Bookmarks.from_raw_values(self._bookmarks) async def _transaction_closed_handler(self): if self._transaction: - self._collect_bookmark(self._transaction._bookmark) + await self._update_bookmark(self._transaction._bookmark) self._transaction = None await self._disconnect() diff --git a/neo4j/_async/work/workspace.py b/neo4j/_async/work/workspace.py index 05b7088d4..6b1f4334c 100644 --- a/neo4j/_async/work/workspace.py +++ b/neo4j/_async/work/workspace.py @@ -20,6 +20,7 @@ import asyncio +from ..._async_compat.util import AsyncUtil from ..._conf import WorkspaceConfig from ..._deadline import Deadline from ..._meta import ( @@ -44,7 +45,9 @@ def __init__(self, pool, config): self._connection_access_mode = None # Sessions are supposed to cache the database on which to operate. self._cached_database = False - self._bookmarks = None + self._bookmarks = () + self._bookmark_manager = None + self._last_from_bookmark_manager = None # Workspace has been closed. self._closed = False @@ -77,6 +80,40 @@ def _set_cached_database(self, database): self._cached_database = True self._config.database = database + async def _get_bookmarks(self, database): + if self._bookmark_manager is not None: + self._bookmarks = tuple( + await AsyncUtil.callback( + self._bookmark_manager.get_bookmarks, database + ) + ) + return self._bookmarks + + async def _get_all_bookmarks(self, must_included_databases): + if self._bookmark_manager is not None: + self._bookmarks = tuple( + await AsyncUtil.callback( + self._bookmark_manager.get_all_bookmarks, + must_included_databases + ) + ) + return self._bookmarks + + async def _update_bookmarks(self, database, new_bookmarks): + if not new_bookmarks: + return + previous_bookmarks = self._bookmarks + self._bookmarks = new_bookmarks + if self._bookmark_manager is None: + return + await self._bookmark_manager.update_bookmarks( + database, previous_bookmarks, new_bookmarks + ) + + async def _update_bookmark(self, bookmark): + if bookmark: + await self._update_bookmarks(self._config.database, (bookmark,)) + async def _connect(self, access_mode, **acquire_kwargs): timeout = Deadline(self._config.session_connection_timeout) if self._connection: diff --git a/neo4j/_conf.py b/neo4j/_conf.py index 0912efe89..a3e5a28a0 100644 --- a/neo4j/_conf.py +++ b/neo4j/_conf.py @@ -429,6 +429,10 @@ class WorkspaceConfig(Config): impersonated_user = None # Note that you need appropriate permissions to do so. + #: Bookmark Manager + bookmark_manager = None + # Specify the bookmark manager to be used for sessions by default. + class SessionConfig(WorkspaceConfig): """ Session configuration. diff --git a/neo4j/_sync/bookmark_manager.py b/neo4j/_sync/bookmark_manager.py new file mode 100644 index 000000000..f2718286f --- /dev/null +++ b/neo4j/_sync/bookmark_manager.py @@ -0,0 +1,86 @@ +# Copyright (c) "Neo4j" +# Neo4j Sweden AB [https://neo4j.com] +# +# This file is part of Neo4j. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import annotations + +import typing as t +from collections import defaultdict + +from .._async_compat.concurrency import ( + CooperativeLock, + Lock, +) +from .._async_compat.util import Util +from ..api import BookmarkManager + + +class Neo4jBookmarkManager(BookmarkManager): + def __init__(self, initial_bookmarks=None, bookmark_supplier=None, + notify_bookmarks=None): + super().__init__() + self._initial_bookmarks = initial_bookmarks + self._bookmark_supplier = bookmark_supplier + self._notify_bookmarks = notify_bookmarks + self._bookmarks = defaultdict(set) + if bookmark_supplier or notify_bookmarks: + self._lock = Lock() + else: + self._lock = CooperativeLock() + + def update_bookmarks( + self, database: str, previous_bookmarks: t.Iterable[str], + new_bookmarks: t.Iterable[str] + ) -> None: + with self._lock: + new_bms = set(new_bookmarks) + if not new_bms: + return + prev_bms = set(previous_bookmarks) + curr_bms = self._bookmarks[database] + curr_bms.difference_update(prev_bms) + curr_bms.update(new_bms) + + if self._notify_bookmarks: + Util.callback( + self._notify_bookmarks, database, tuple(curr_bms) + ) + + def _get_bookmarks(self, database: str) -> t.Set[str]: + bms = self._bookmarks[database] + if self._bookmark_supplier: + extra_bms = Util.callback( + self._bookmark_supplier, database + ) + if extra_bms is not None: + bms &= set(extra_bms) + return bms + + def get_bookmarks(self, database: str) -> t.Set[str]: + with self._lock: + return self._get_bookmarks(database) + + def get_all_bookmarks( + self, must_included_databases: t.Iterable[str] + ) -> t.Set[str]: + with self._lock: + bms = set() + databases = (set(must_included_databases) + | set(self._bookmarks.keys())) + for database in databases: + bms.update(self._get_bookmarks(database)) + return bms diff --git a/neo4j/_sync/driver.py b/neo4j/_sync/driver.py index 5834b46ba..cf1090664 100644 --- a/neo4j/_sync/driver.py +++ b/neo4j/_sync/driver.py @@ -44,6 +44,7 @@ from ..addressing import Address from ..api import ( Auth, + BookmarkManager, Bookmarks, DRIVER_BOLT, DRIVER_NEO4J, @@ -62,9 +63,14 @@ URI_SCHEME_NEO4J_SECURE, URI_SCHEME_NEO4J_SELF_SIGNED_CERTIFICATE, ) +from .bookmark_manager import Neo4jBookmarkManager from .work import Session +_T_BmSupplier = t.Callable[[str], t.Union[Bookmarks, t.Union[Bookmarks]]] +_T_NotifyBm = t.Callable[[str, Bookmarks], t.Union[None, t.Union[None]]] + + class GraphDatabase: """Accessor for :class:`neo4j.Driver` construction. """ @@ -94,6 +100,8 @@ def driver( ssl_context: ssl.SSLContext = ..., user_agent: str = ..., keep_alive: bool = ..., + bookmark_manager: t.Union[BookmarkManager, + BookmarkManager, None] = ..., # undocumented/unsupported options # they may be change or removed any time without prior notice @@ -208,6 +216,36 @@ def driver(cls, uri, *, auth=None, **config) -> Driver: return cls.neo4j_driver(parsed.netloc, auth=auth, routing_context=routing_context, **config) + @classmethod + def bookmark_manager( + cls, initial_bookmarks: Bookmarks = None, + bookmark_supplier: _T_BmSupplier = None, + notify_bookmarks: _T_NotifyBm = None + ) -> BookmarkManager: + """Create a default :class:`BookmarkManager`. + + :param initial_bookmarks: + The initial set of bookmarks. The default bookmark manager will + seed the set of bookmarks for each database with this value. + :param bookmark_supplier: + Function which will be called every time the default bookmark + manager's method :meth:`.BookmarkManager.get_bookmarks` + gets called. The result of ``bookmark_supplier`` will be + concatenated with the internal set of bookmarks and used to + configure the session in creation. + :param notify_bookmarks: + Function which will be called whenever the set of bookmarks + handled by the bookmark manager gets updated with the new + internal bookmark set. + + :returns: A default implementation of :class:`BookmarkManager`. + """ + return Neo4jBookmarkManager( + initial_bookmarks=initial_bookmarks, + bookmark_supplier=bookmark_supplier, + notify_bookmarks=notify_bookmarks + ) + @classmethod def bolt_driver(cls, target, *, auth=None, **config): """ Create a driver for direct Bolt server access that uses @@ -353,6 +391,8 @@ def session( initial_retry_delay: float = ..., retry_delay_multiplier: float = ..., retry_delay_jitter_factor: float = ..., + bookmark_manager: t.Union[BookmarkManager, + BookmarkManager, None] = ..., ) -> Session: ... @@ -394,6 +434,8 @@ def verify_connectivity( initial_retry_delay: float = ..., retry_delay_multiplier: float = ..., retry_delay_jitter_factor: float = ..., + bookmark_manager: t.Union[BookmarkManager, + BookmarkManager, None] = ..., ) -> None: ... @@ -456,6 +498,8 @@ def get_server_info( initial_retry_delay: float = ..., retry_delay_multiplier: float = ..., retry_delay_jitter_factor: float = ..., + bookmark_manager: t.Union[BookmarkManager, + BookmarkManager, None] = ..., ) -> ServerInfo: ... diff --git a/neo4j/_sync/work/session.py b/neo4j/_sync/work/session.py index 9e0102bdd..e712df19e 100644 --- a/neo4j/_sync/work/session.py +++ b/neo4j/_sync/work/session.py @@ -100,9 +100,12 @@ class Session(Workspace): _state_failed = False def __init__(self, pool, session_config): - super().__init__(pool, session_config) assert isinstance(session_config, SessionConfig) - self._bookmarks = self._prepare_bookmarks(session_config.bookmarks) + super().__init__(pool, session_config) + if session_config.bookmarks is not None: + self._bookmarks = self._prepare_bookmarks(session_config.bookmarks) + else: + self._bookmark_manager = session_config.bookmark_manager def __enter__(self) -> Session: return self @@ -147,10 +150,6 @@ def _disconnect(self, sync=False): self._handle_cancellation(message="_disconnect") raise - def _collect_bookmark(self, bookmark): - if bookmark: - self._bookmarks = bookmark, - def _handle_cancellation(self, message="General"): self._transaction = None self._auto_result = None @@ -165,7 +164,7 @@ def _handle_cancellation(self, message="General"): def _result_closed(self): if self._auto_result: - self._collect_bookmark(self._auto_result._bookmark) + self._update_bookmark(self._auto_result._bookmark) self._auto_result = None self._disconnect() @@ -196,7 +195,7 @@ def close(self) -> None: if self._state_failed is False: try: self._auto_result.consume() - self._collect_bookmark(self._auto_result._bookmark) + self._update_bookmark(self._auto_result._bookmark) except Exception as error: # TODO: Investigate potential non graceful close states self._auto_result = None @@ -336,7 +335,7 @@ def last_bookmark(self) -> t.Optional[str]: self._auto_result.consume() if self._transaction and self._transaction._closed: - self._collect_bookmark(self._transaction._bookmark) + self._update_bookmark(self._transaction._bookmark) self._transaction = None if self._bookmarks: @@ -377,14 +376,14 @@ def last_bookmarks(self) -> Bookmarks: self._auto_result.consume() if self._transaction and self._transaction._closed(): - self._collect_bookmark(self._transaction._bookmark) + self._update_bookmark(self._transaction._bookmark) self._transaction = None return Bookmarks.from_raw_values(self._bookmarks) def _transaction_closed_handler(self): if self._transaction: - self._collect_bookmark(self._transaction._bookmark) + self._update_bookmark(self._transaction._bookmark) self._transaction = None self._disconnect() diff --git a/neo4j/_sync/work/workspace.py b/neo4j/_sync/work/workspace.py index 7c3f5865b..dfc2bee23 100644 --- a/neo4j/_sync/work/workspace.py +++ b/neo4j/_sync/work/workspace.py @@ -20,6 +20,7 @@ import asyncio +from ..._async_compat.util import Util from ..._conf import WorkspaceConfig from ..._deadline import Deadline from ..._meta import ( @@ -44,7 +45,9 @@ def __init__(self, pool, config): self._connection_access_mode = None # Sessions are supposed to cache the database on which to operate. self._cached_database = False - self._bookmarks = None + self._bookmarks = () + self._bookmark_manager = None + self._last_from_bookmark_manager = None # Workspace has been closed. self._closed = False @@ -77,6 +80,40 @@ def _set_cached_database(self, database): self._cached_database = True self._config.database = database + def _get_bookmarks(self, database): + if self._bookmark_manager is not None: + self._bookmarks = tuple( + Util.callback( + self._bookmark_manager.get_bookmarks, database + ) + ) + return self._bookmarks + + def _get_all_bookmarks(self, must_included_databases): + if self._bookmark_manager is not None: + self._bookmarks = tuple( + Util.callback( + self._bookmark_manager.get_all_bookmarks, + must_included_databases + ) + ) + return self._bookmarks + + def _update_bookmarks(self, database, new_bookmarks): + if not new_bookmarks: + return + previous_bookmarks = self._bookmarks + self._bookmarks = new_bookmarks + if self._bookmark_manager is None: + return + self._bookmark_manager.update_bookmarks( + database, previous_bookmarks, new_bookmarks + ) + + def _update_bookmark(self, bookmark): + if bookmark: + self._update_bookmarks(self._config.database, (bookmark,)) + def _connect(self, access_mode, **acquire_kwargs): timeout = Deadline(self._config.session_connection_timeout) if self._connection: diff --git a/neo4j/api.py b/neo4j/api.py index 7db50c907..1aca15aaa 100644 --- a/neo4j/api.py +++ b/neo4j/api.py @@ -20,6 +20,7 @@ from __future__ import annotations +import abc import typing as t from urllib.parse import ( parse_qs, @@ -363,6 +364,93 @@ def from_bytes(cls, b: bytes) -> Version: return Version(b[-1], b[-2]) +class BookmarkManager(abc.ABC): + """Class to manage bookmarks throughout the driver's lifetime. + + Neo4j clusters are eventually consistent, meaning that there is no + guarantee a query will be able to read changes made by a previous query. + For cases where such a guarantee is necessary, the server provides + bookmarks to the client. A bookmark is an abstract token that represents + some state of the database. By passing one or multiple bookmarks along + with a query, the server will make sure that the query will not get + executed before the represented state(s) (or a later state) have been + established. + + The bookmark manager is an interface used by the driver for keeping + track of the bookmarks and this way keeping sessions automatically + consistent. + + .. note:: + All methods must be concurrency safe. + """ + + @abc.abstractmethod + def update_bookmarks( + self, database: str, previous_bookmarks: t.Iterable[str], + new_bookmarks: t.Iterable[str] + ) -> None: + """Handle bookmark updates. + + :param database: + The database which the bookmarks belong to + :param previous_bookmarks: + The bookmarks used at the start of a transaction + :param new_bookmarks: + The new bookmarks retrieved at the end of a transaction + """ + ... + + @abc.abstractmethod + def get_bookmarks(self, database: str) -> t.Collection[str]: + """Return the bookmarks for a given database. + + :param database: The database which the bookmarks belong to + + :returns: The bookmarks for the given database + """ + ... + + @abc.abstractmethod + def get_all_bookmarks( + self, must_included_databases: t.Iterable[str] + ) -> t.Collection[str]: + """Return all bookmarks. + + The prototypical implementation of this method iterates over all known + databases plus the ones provided in the `must_included_databases` + parameter and calls :meth:`get_bookmarks` for each of them. It then + returns the union of all the bookmarks. + + :param must_included_databases: + The databases which must be included in the result even if they + don't have been initialized yet. + + :returns: The collected bookmarks. + """ + ... + + +class AsyncBookmarkManager(abc.ABC): + """Same as :class:`BookmarkManager` but with async methods.""" + + @abc.abstractmethod + async def update_bookmarks( + self, database: str, previous_bookmarks: t.Iterable[str], + new_bookmarks: t.Iterable[str] + ) -> None: + ... + + @abc.abstractmethod + async def get_bookmarks(self, database: str) -> t.Collection[str]: + ... + + @abc.abstractmethod + async def get_all_bookmarks( + self, must_included_databases: t.Iterable[str] + ) -> t.Collection[str]: + ... + + def parse_neo4j_uri(uri): parsed = urlparse(uri) From 1ecd68f1802b14a912c330dd6a5ed4196150bb25 Mon Sep 17 00:00:00 2001 From: Rouven Bauer Date: Wed, 3 Aug 2022 17:03:09 +0200 Subject: [PATCH 02/26] Fix unit tests for config --- tests/unit/common/test_conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/common/test_conf.py b/tests/unit/common/test_conf.py index 684dbaa35..b45cf272f 100644 --- a/tests/unit/common/test_conf.py +++ b/tests/unit/common/test_conf.py @@ -68,6 +68,7 @@ "database": None, "impersonated_user": None, "fetch_size": 100, + "bookmark_manager": object(), } From 32273f98693b179aa5ee210fa4a700c43c77f452 Mon Sep 17 00:00:00 2001 From: Rouven Bauer Date: Thu, 4 Aug 2022 12:57:30 +0200 Subject: [PATCH 03/26] TestKit backend support for bookmark manager --- testkitbackend/_async/requests.py | 17 ++++++++++++++++- testkitbackend/_sync/requests.py | 17 ++++++++++++++++- testkitbackend/test_config.json | 1 + 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/testkitbackend/_async/requests.py b/testkitbackend/_async/requests.py index 0ea955cb6..1c37e5fe3 100644 --- a/testkitbackend/_async/requests.py +++ b/testkitbackend/_async/requests.py @@ -144,6 +144,21 @@ async def NewDriver(backend, data): for cert in data["trustedCertificates"]) kwargs["trusted_certificates"] = neo4j.TrustCustomCAs(*cert_paths) data.mark_item_as_read_if_equals("livenessCheckTimeoutMs", None) + bookmark_manager_config = data.get("bookmarkManager", {}) + if bookmark_manager_config: + bookmark_manager_config.mark_item_as_read_if_equals( + "bookmarkSupplier", False + ) + bookmark_manager_config.mark_item_as_read_if_equals( + "notifyBookmarks", False + ) + bookmark_manager_config.mark_item_as_read("initialBookmarks", + recursive=True) + kwargs["bookmark_manager"] = neo4j.AsyncGraphDatabase.bookmark_manager( + initial_bookmarks=bookmark_manager_config.get("initialBookmarks"), + bookmark_supplier=None, + notify_bookmarks=None, + ) data.mark_item_as_read("domainNameResolverRegistered") driver = neo4j.AsyncGraphDatabase.driver( @@ -277,7 +292,7 @@ async def NewSession(backend, data): else: raise ValueError("Unknown access mode:" + access_mode) bookmarks = None - if "bookmarks" in data and data["bookmarks"]: + if "bookmarks" in data and data["bookmarks"] is not None: bookmarks = neo4j.Bookmarks.from_raw_values(data["bookmarks"]) config = { "default_access_mode": access_mode, diff --git a/testkitbackend/_sync/requests.py b/testkitbackend/_sync/requests.py index 776cfa95a..4af03608f 100644 --- a/testkitbackend/_sync/requests.py +++ b/testkitbackend/_sync/requests.py @@ -144,6 +144,21 @@ def NewDriver(backend, data): for cert in data["trustedCertificates"]) kwargs["trusted_certificates"] = neo4j.TrustCustomCAs(*cert_paths) data.mark_item_as_read_if_equals("livenessCheckTimeoutMs", None) + bookmark_manager_config = data.get("bookmarkManager", {}) + if bookmark_manager_config: + bookmark_manager_config.mark_item_as_read_if_equals( + "bookmarkSupplier", False + ) + bookmark_manager_config.mark_item_as_read_if_equals( + "notifyBookmarks", False + ) + bookmark_manager_config.mark_item_as_read("initialBookmarks", + recursive=True) + kwargs["bookmark_manager"] = neo4j.GraphDatabase.bookmark_manager( + initial_bookmarks=bookmark_manager_config.get("initialBookmarks"), + bookmark_supplier=None, + notify_bookmarks=None, + ) data.mark_item_as_read("domainNameResolverRegistered") driver = neo4j.GraphDatabase.driver( @@ -277,7 +292,7 @@ def NewSession(backend, data): else: raise ValueError("Unknown access mode:" + access_mode) bookmarks = None - if "bookmarks" in data and data["bookmarks"]: + if "bookmarks" in data and data["bookmarks"] is not None: bookmarks = neo4j.Bookmarks.from_raw_values(data["bookmarks"]) config = { "default_access_mode": access_mode, diff --git a/testkitbackend/test_config.json b/testkitbackend/test_config.json index da189e1cf..6a1a9ff0f 100644 --- a/testkitbackend/test_config.json +++ b/testkitbackend/test_config.json @@ -16,6 +16,7 @@ "test_subtest_skips.tz_id" }, "features": { + "Feature:API:BookmarkManager": true, "Feature:API:ConnectionAcquisitionTimeout": true, "Feature:API:Driver:GetServerInfo": true, "Feature:API:Driver.IsEncrypted": true, From 2279b6785ffb747ac834d0e00e006ecc8e107d4f Mon Sep 17 00:00:00 2001 From: Rouven Bauer Date: Thu, 4 Aug 2022 13:41:42 +0200 Subject: [PATCH 04/26] Fix bookmark manager * respect database reported in summary * fix lock usage * other small fixes --- docs/source/api.rst | 1 + neo4j/_async/bookmark_manager.py | 14 +++++++++----- neo4j/_async/driver.py | 8 ++++++-- neo4j/_async/work/result.py | 5 +++++ neo4j/_async/work/session.py | 23 ++++++++++++++++------- neo4j/_async/work/transaction.py | 3 +++ neo4j/_async/work/workspace.py | 26 ++++++++++++++++---------- neo4j/_async_compat/concurrency.py | 17 ++++++++++++++--- neo4j/_sync/bookmark_manager.py | 8 ++++++-- neo4j/_sync/driver.py | 8 ++++++-- neo4j/_sync/work/result.py | 5 +++++ neo4j/_sync/work/session.py | 23 ++++++++++++++++------- neo4j/_sync/work/transaction.py | 3 +++ neo4j/_sync/work/workspace.py | 26 ++++++++++++++++---------- neo4j/api.py | 13 +++++++++++++ 15 files changed, 135 insertions(+), 48 deletions(-) diff --git a/docs/source/api.rst b/docs/source/api.rst index 0d6b6831e..fadef6a3f 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -1522,6 +1522,7 @@ Bookmarks .. autoclass:: neo4j.Bookmarks :members: + :special-members: __bool__, __add__, __iter__ .. autoclass:: neo4j.Bookmark :members: diff --git a/neo4j/_async/bookmark_manager.py b/neo4j/_async/bookmark_manager.py index ee0545bd7..c068595e7 100644 --- a/neo4j/_async/bookmark_manager.py +++ b/neo4j/_async/bookmark_manager.py @@ -33,10 +33,14 @@ class AsyncNeo4jBookmarkManager(AsyncBookmarkManager): def __init__(self, initial_bookmarks=None, bookmark_supplier=None, notify_bookmarks=None): super().__init__() - self._initial_bookmarks = initial_bookmarks self._bookmark_supplier = bookmark_supplier self._notify_bookmarks = notify_bookmarks - self._bookmarks = defaultdict(set) + if initial_bookmarks is None: + initial_bookmarks = {} + self._bookmarks = defaultdict( + set, ((k, set(v)) for k, v in initial_bookmarks.items()) + ) + self._lock: t.Union[AsyncLock, AsyncCooperativeLock] if bookmark_supplier or notify_bookmarks: self._lock = AsyncLock() else: @@ -46,7 +50,7 @@ async def update_bookmarks( self, database: str, previous_bookmarks: t.Iterable[str], new_bookmarks: t.Iterable[str] ) -> None: - with self._lock: + async with self._lock: new_bms = set(new_bookmarks) if not new_bms: return @@ -71,13 +75,13 @@ async def _get_bookmarks(self, database: str) -> t.Set[str]: return bms async def get_bookmarks(self, database: str) -> t.Set[str]: - with self._lock: + async with self._lock: return await self._get_bookmarks(database) async def get_all_bookmarks( self, must_included_databases: t.Iterable[str] ) -> t.Set[str]: - with self._lock: + async with self._lock: bms = set() databases = (set(must_included_databases) | set(self._bookmarks.keys())) diff --git a/neo4j/_async/driver.py b/neo4j/_async/driver.py index 0b21d4d02..7bb16248e 100644 --- a/neo4j/_async/driver.py +++ b/neo4j/_async/driver.py @@ -219,7 +219,9 @@ def driver(cls, uri, *, auth=None, **config) -> AsyncDriver: @classmethod def bookmark_manager( - cls, initial_bookmarks: Bookmarks = None, + cls, + initial_bookmarks: t.Mapping[str, t.Union[Bookmarks, + t.Iterable[str]]] = None, bookmark_supplier: _T_BmSupplier = None, notify_bookmarks: _T_NotifyBm = None ) -> AsyncBookmarkManager: @@ -227,7 +229,9 @@ def bookmark_manager( :param initial_bookmarks: The initial set of bookmarks. The default bookmark manager will - seed the set of bookmarks for each database with this value. + use this to initialize its internal bookmarks per database. + If present, this parameter must be a mapping of database names + to :class:`.Bookmarks` or an iterable of raw bookmark values (str). :param bookmark_supplier: Function which will be called every time the default bookmark manager's method :meth:`.AsyncBookmarkManager.get_bookmarks` diff --git a/neo4j/_async/work/result.py b/neo4j/_async/work/result.py index 5e8705850..e1e3d08ed 100644 --- a/neo4j/_async/work/result.py +++ b/neo4j/_async/work/result.py @@ -80,6 +80,7 @@ def __init__(self, connection, fetch_size, on_closed, on_error): self._keys = None self._record_buffer = deque() self._summary = None + self._database = None self._bookmark = None self._raw_qid = -1 self._fetch_size = fetch_size @@ -127,7 +128,9 @@ async def _run( "query": query_text, "parameters": parameters, "server": self._connection.server_info, + "database": db, } + self._database = db def on_attached(metadata): self._metadata.update(metadata) @@ -189,6 +192,7 @@ def on_success(summary_metadata): return self._metadata.update(summary_metadata) self._bookmark = summary_metadata.get("bookmark") + self._database = summary_metadata.get("db", self._database) self._connection.pull( n=self._fetch_size, @@ -220,6 +224,7 @@ def on_success(summary_metadata): self._discarding = False self._metadata.update(summary_metadata) self._bookmark = summary_metadata.get("bookmark") + self._database = summary_metadata.get("db", self._database) # This was the last page received, discard the rest self._connection.discard( diff --git a/neo4j/_async/work/session.py b/neo4j/_async/work/session.py index 06301a715..81e5e58f1 100644 --- a/neo4j/_async/work/session.py +++ b/neo4j/_async/work/session.py @@ -164,7 +164,8 @@ def _handle_cancellation(self, message="General"): async def _result_closed(self): if self._auto_result: - await self._update_bookmark(self._auto_result._bookmark) + await self._update_bookmark(self._auto_result._database, + self._auto_result._bookmark) self._auto_result = None await self._disconnect() @@ -195,7 +196,10 @@ async def close(self) -> None: if self._state_failed is False: try: await self._auto_result.consume() - await self._update_bookmark(self._auto_result._bookmark) + await self._update_bookmark( + self._auto_result._database, + self._auto_result._bookmark + ) except Exception as error: # TODO: Investigate potential non graceful close states self._auto_result = None @@ -301,10 +305,11 @@ async def run( cx, self._config.fetch_size, self._result_closed, self._result_error ) + bookmarks = await self._get_all_bookmarks() await self._auto_result._run( query, parameters, self._config.database, self._config.impersonated_user, self._config.default_access_mode, - self._bookmarks, **kwargs + bookmarks, **kwargs ) return self._auto_result @@ -335,7 +340,8 @@ async def last_bookmark(self) -> t.Optional[str]: await self._auto_result.consume() if self._transaction and self._transaction._closed: - await self._update_bookmark(self._transaction._bookmark) + await self._update_bookmark(self._transaction._database, + self._transaction._bookmark) self._transaction = None if self._bookmarks: @@ -376,14 +382,16 @@ async def last_bookmarks(self) -> Bookmarks: await self._auto_result.consume() if self._transaction and self._transaction._closed(): - await self._update_bookmark(self._transaction._bookmark) + await self._update_bookmark(self._transaction._database, + self._transaction._bookmark) self._transaction = None return Bookmarks.from_raw_values(self._bookmarks) async def _transaction_closed_handler(self): if self._transaction: - await self._update_bookmark(self._transaction._bookmark) + await self._update_bookmark(self._transaction._database, + self._transaction._bookmark) self._transaction = None await self._disconnect() @@ -407,9 +415,10 @@ async def _open_transaction( self._transaction_error_handler, self._transaction_cancel_handler ) + bookmarks = await self._get_all_bookmarks() await self._transaction._begin( self._config.database, self._config.impersonated_user, - self._bookmarks, access_mode, metadata, timeout + bookmarks, access_mode, metadata, timeout ) async def begin_transaction( diff --git a/neo4j/_async/work/transaction.py b/neo4j/_async/work/transaction.py index 11a496ac9..eee74c86c 100644 --- a/neo4j/_async/work/transaction.py +++ b/neo4j/_async/work/transaction.py @@ -44,6 +44,7 @@ def __init__(self, connection, fetch_size, on_closed, on_error, connection, self._error_handler ) self._bookmark = None + self._database = None self._results = [] self._closed_flag = False self._last_error = None @@ -69,6 +70,7 @@ async def _exit(self, exception_type, exception_value, traceback): async def _begin( self, database, imp_user, bookmarks, access_mode, metadata, timeout ): + self._database = database self._connection.begin( bookmarks=bookmarks, metadata=metadata, timeout=timeout, mode=access_mode, db=database, imp_user=imp_user @@ -168,6 +170,7 @@ async def _commit(self): await self._connection.send_all() await self._connection.fetch_all() self._bookmark = metadata.get("bookmark") + self._database = metadata.get("db", self._database) except asyncio.CancelledError: self._on_cancel() raise diff --git a/neo4j/_async/work/workspace.py b/neo4j/_async/work/workspace.py index 6b1f4334c..37af39feb 100644 --- a/neo4j/_async/work/workspace.py +++ b/neo4j/_async/work/workspace.py @@ -80,16 +80,19 @@ def _set_cached_database(self, database): self._cached_database = True self._config.database = database - async def _get_bookmarks(self, database): + async def _get_system_bookmarks(self): if self._bookmark_manager is not None: self._bookmarks = tuple( await AsyncUtil.callback( - self._bookmark_manager.get_bookmarks, database + self._bookmark_manager.get_bookmarks, "system" ) ) return self._bookmarks - async def _get_all_bookmarks(self, must_included_databases): + async def _get_all_bookmarks(self): + must_included_databases = ["system"] + if self._config.database: + must_included_databases.append(self._config.database) if self._bookmark_manager is not None: self._bookmarks = tuple( await AsyncUtil.callback( @@ -100,19 +103,21 @@ async def _get_all_bookmarks(self, must_included_databases): return self._bookmarks async def _update_bookmarks(self, database, new_bookmarks): - if not new_bookmarks: - return previous_bookmarks = self._bookmarks self._bookmarks = new_bookmarks if self._bookmark_manager is None: return - await self._bookmark_manager.update_bookmarks( + await AsyncUtil.callback( + self._bookmark_manager.update_bookmarks, database, previous_bookmarks, new_bookmarks ) - async def _update_bookmark(self, bookmark): - if bookmark: - await self._update_bookmarks(self._config.database, (bookmark,)) + async def _update_bookmark(self, database, bookmark): + if not bookmark: + return + if not database: + database = self._config.database + await self._update_bookmarks(database, (bookmark,)) async def _connect(self, access_mode, **acquire_kwargs): timeout = Deadline(self._config.session_connection_timeout) @@ -133,10 +138,11 @@ async def _connect(self, access_mode, **acquire_kwargs): # to try to fetch the home database. If provided by the server, # we shall use this database explicitly for all subsequent # actions within this session. + bookmarks = await self._get_system_bookmarks() await self._pool.update_routing_table( database=self._config.database, imp_user=self._config.impersonated_user, - bookmarks=self._bookmarks, + bookmarks=bookmarks, timeout=timeout, database_callback=self._set_cached_database ) diff --git a/neo4j/_async_compat/concurrency.py b/neo4j/_async_compat/concurrency.py index 3f149a668..96611f72a 100644 --- a/neo4j/_async_compat/concurrency.py +++ b/neo4j/_async_compat/concurrency.py @@ -16,10 +16,13 @@ # limitations under the License. +from __future__ import annotations + import asyncio import collections import re import threading +import typing as t from neo4j._async_compat.shims import wait_for @@ -211,6 +214,12 @@ def release(self): def __exit__(self, t, v, tb): self.release() + async def __aenter__(self): + return self.__enter__() + + async def __aexit__(self, t, v, tb): + self.__exit__(t, v, tb) + class AsyncCooperativeRLock: """Reentrant lock placeholder for cooperative asyncio Python. @@ -423,6 +432,8 @@ def notify_all(self): self.notify(len(self._waiters)) -Condition = threading.Condition -CooperativeLock = Lock = threading.Lock -CooperativeRLock = RLock = threading.RLock +Condition: t.TypeAlias = threading.Condition +CooperativeLock: t.TypeAlias = threading.Lock +Lock: t.TypeAlias = threading.Lock +CooperativeRLock: t.TypeAlias = threading.RLock +RLock: t.TypeAlias = threading.RLock diff --git a/neo4j/_sync/bookmark_manager.py b/neo4j/_sync/bookmark_manager.py index f2718286f..5b7560e1a 100644 --- a/neo4j/_sync/bookmark_manager.py +++ b/neo4j/_sync/bookmark_manager.py @@ -33,10 +33,14 @@ class Neo4jBookmarkManager(BookmarkManager): def __init__(self, initial_bookmarks=None, bookmark_supplier=None, notify_bookmarks=None): super().__init__() - self._initial_bookmarks = initial_bookmarks self._bookmark_supplier = bookmark_supplier self._notify_bookmarks = notify_bookmarks - self._bookmarks = defaultdict(set) + if initial_bookmarks is None: + initial_bookmarks = {} + self._bookmarks = defaultdict( + set, ((k, set(v)) for k, v in initial_bookmarks.items()) + ) + self._lock: t.Union[Lock, CooperativeLock] if bookmark_supplier or notify_bookmarks: self._lock = Lock() else: diff --git a/neo4j/_sync/driver.py b/neo4j/_sync/driver.py index cf1090664..a26e30349 100644 --- a/neo4j/_sync/driver.py +++ b/neo4j/_sync/driver.py @@ -218,7 +218,9 @@ def driver(cls, uri, *, auth=None, **config) -> Driver: @classmethod def bookmark_manager( - cls, initial_bookmarks: Bookmarks = None, + cls, + initial_bookmarks: t.Mapping[str, t.Union[Bookmarks, + t.Iterable[str]]] = None, bookmark_supplier: _T_BmSupplier = None, notify_bookmarks: _T_NotifyBm = None ) -> BookmarkManager: @@ -226,7 +228,9 @@ def bookmark_manager( :param initial_bookmarks: The initial set of bookmarks. The default bookmark manager will - seed the set of bookmarks for each database with this value. + use this to initialize its internal bookmarks per database. + If present, this parameter must be a mapping of database names + to :class:`.Bookmarks` or an iterable of raw bookmark values (str). :param bookmark_supplier: Function which will be called every time the default bookmark manager's method :meth:`.BookmarkManager.get_bookmarks` diff --git a/neo4j/_sync/work/result.py b/neo4j/_sync/work/result.py index f1b7cc904..e987a4186 100644 --- a/neo4j/_sync/work/result.py +++ b/neo4j/_sync/work/result.py @@ -80,6 +80,7 @@ def __init__(self, connection, fetch_size, on_closed, on_error): self._keys = None self._record_buffer = deque() self._summary = None + self._database = None self._bookmark = None self._raw_qid = -1 self._fetch_size = fetch_size @@ -127,7 +128,9 @@ def _run( "query": query_text, "parameters": parameters, "server": self._connection.server_info, + "database": db, } + self._database = db def on_attached(metadata): self._metadata.update(metadata) @@ -189,6 +192,7 @@ def on_success(summary_metadata): return self._metadata.update(summary_metadata) self._bookmark = summary_metadata.get("bookmark") + self._database = summary_metadata.get("db", self._database) self._connection.pull( n=self._fetch_size, @@ -220,6 +224,7 @@ def on_success(summary_metadata): self._discarding = False self._metadata.update(summary_metadata) self._bookmark = summary_metadata.get("bookmark") + self._database = summary_metadata.get("db", self._database) # This was the last page received, discard the rest self._connection.discard( diff --git a/neo4j/_sync/work/session.py b/neo4j/_sync/work/session.py index e712df19e..3fb01f86a 100644 --- a/neo4j/_sync/work/session.py +++ b/neo4j/_sync/work/session.py @@ -164,7 +164,8 @@ def _handle_cancellation(self, message="General"): def _result_closed(self): if self._auto_result: - self._update_bookmark(self._auto_result._bookmark) + self._update_bookmark(self._auto_result._database, + self._auto_result._bookmark) self._auto_result = None self._disconnect() @@ -195,7 +196,10 @@ def close(self) -> None: if self._state_failed is False: try: self._auto_result.consume() - self._update_bookmark(self._auto_result._bookmark) + self._update_bookmark( + self._auto_result._database, + self._auto_result._bookmark + ) except Exception as error: # TODO: Investigate potential non graceful close states self._auto_result = None @@ -301,10 +305,11 @@ def run( cx, self._config.fetch_size, self._result_closed, self._result_error ) + bookmarks = self._get_all_bookmarks() self._auto_result._run( query, parameters, self._config.database, self._config.impersonated_user, self._config.default_access_mode, - self._bookmarks, **kwargs + bookmarks, **kwargs ) return self._auto_result @@ -335,7 +340,8 @@ def last_bookmark(self) -> t.Optional[str]: self._auto_result.consume() if self._transaction and self._transaction._closed: - self._update_bookmark(self._transaction._bookmark) + self._update_bookmark(self._transaction._database, + self._transaction._bookmark) self._transaction = None if self._bookmarks: @@ -376,14 +382,16 @@ def last_bookmarks(self) -> Bookmarks: self._auto_result.consume() if self._transaction and self._transaction._closed(): - self._update_bookmark(self._transaction._bookmark) + self._update_bookmark(self._transaction._database, + self._transaction._bookmark) self._transaction = None return Bookmarks.from_raw_values(self._bookmarks) def _transaction_closed_handler(self): if self._transaction: - self._update_bookmark(self._transaction._bookmark) + self._update_bookmark(self._transaction._database, + self._transaction._bookmark) self._transaction = None self._disconnect() @@ -407,9 +415,10 @@ def _open_transaction( self._transaction_error_handler, self._transaction_cancel_handler ) + bookmarks = self._get_all_bookmarks() self._transaction._begin( self._config.database, self._config.impersonated_user, - self._bookmarks, access_mode, metadata, timeout + bookmarks, access_mode, metadata, timeout ) def begin_transaction( diff --git a/neo4j/_sync/work/transaction.py b/neo4j/_sync/work/transaction.py index 8a297dd07..41c1fd895 100644 --- a/neo4j/_sync/work/transaction.py +++ b/neo4j/_sync/work/transaction.py @@ -44,6 +44,7 @@ def __init__(self, connection, fetch_size, on_closed, on_error, connection, self._error_handler ) self._bookmark = None + self._database = None self._results = [] self._closed_flag = False self._last_error = None @@ -69,6 +70,7 @@ def _exit(self, exception_type, exception_value, traceback): def _begin( self, database, imp_user, bookmarks, access_mode, metadata, timeout ): + self._database = database self._connection.begin( bookmarks=bookmarks, metadata=metadata, timeout=timeout, mode=access_mode, db=database, imp_user=imp_user @@ -168,6 +170,7 @@ def _commit(self): self._connection.send_all() self._connection.fetch_all() self._bookmark = metadata.get("bookmark") + self._database = metadata.get("db", self._database) except asyncio.CancelledError: self._on_cancel() raise diff --git a/neo4j/_sync/work/workspace.py b/neo4j/_sync/work/workspace.py index dfc2bee23..4df469b80 100644 --- a/neo4j/_sync/work/workspace.py +++ b/neo4j/_sync/work/workspace.py @@ -80,16 +80,19 @@ def _set_cached_database(self, database): self._cached_database = True self._config.database = database - def _get_bookmarks(self, database): + def _get_system_bookmarks(self): if self._bookmark_manager is not None: self._bookmarks = tuple( Util.callback( - self._bookmark_manager.get_bookmarks, database + self._bookmark_manager.get_bookmarks, "system" ) ) return self._bookmarks - def _get_all_bookmarks(self, must_included_databases): + def _get_all_bookmarks(self): + must_included_databases = ["system"] + if self._config.database: + must_included_databases.append(self._config.database) if self._bookmark_manager is not None: self._bookmarks = tuple( Util.callback( @@ -100,19 +103,21 @@ def _get_all_bookmarks(self, must_included_databases): return self._bookmarks def _update_bookmarks(self, database, new_bookmarks): - if not new_bookmarks: - return previous_bookmarks = self._bookmarks self._bookmarks = new_bookmarks if self._bookmark_manager is None: return - self._bookmark_manager.update_bookmarks( + Util.callback( + self._bookmark_manager.update_bookmarks, database, previous_bookmarks, new_bookmarks ) - def _update_bookmark(self, bookmark): - if bookmark: - self._update_bookmarks(self._config.database, (bookmark,)) + def _update_bookmark(self, database, bookmark): + if not bookmark: + return + if not database: + database = self._config.database + self._update_bookmarks(database, (bookmark,)) def _connect(self, access_mode, **acquire_kwargs): timeout = Deadline(self._config.session_connection_timeout) @@ -133,10 +138,11 @@ def _connect(self, access_mode, **acquire_kwargs): # to try to fetch the home database. If provided by the server, # we shall use this database explicitly for all subsequent # actions within this session. + bookmarks = self._get_system_bookmarks() self._pool.update_routing_table( database=self._config.database, imp_user=self._config.impersonated_user, - bookmarks=self._bookmarks, + bookmarks=bookmarks, timeout=timeout, database_callback=self._set_cached_database ) diff --git a/neo4j/api.py b/neo4j/api.py index 1aca15aaa..a11eec821 100644 --- a/neo4j/api.py +++ b/neo4j/api.py @@ -236,9 +236,11 @@ def __repr__(self) -> str: ) def __bool__(self) -> bool: + """True if there are bookmarks in the container.""" return bool(self._raw_values) def __add__(self, other: Bookmarks) -> Bookmarks: + """Add multiple containers together.""" if isinstance(other, Bookmarks): if not other: return self @@ -247,6 +249,17 @@ def __add__(self, other: Bookmarks) -> Bookmarks: return ret return NotImplemented + def __iter__(self) -> t.Iterator[str]: + """Iterate over the raw bookmark values. + + This is equivalent to:: + + bookmarks.raw_values.__iter__() + + :return: iterator over the raw bookmark values + """ + return iter(self._raw_values) + @property def raw_values(self) -> t.FrozenSet[str]: """The raw bookmark values. From e696f07500adbd211cb7470a1e86c61fbfbc9051 Mon Sep 17 00:00:00 2001 From: Rouven Bauer Date: Fri, 5 Aug 2022 11:30:17 +0200 Subject: [PATCH 05/26] Add support for `ignore_bookmark_manager` session config option --- neo4j/_async/driver.py | 1 + neo4j/_async/work/session.py | 20 ++-------------- neo4j/_async/work/workspace.py | 40 ++++++++++++++++++++++++------- neo4j/_conf.py | 3 +++ neo4j/_sync/driver.py | 1 + neo4j/_sync/work/session.py | 20 ++-------------- neo4j/_sync/work/workspace.py | 40 ++++++++++++++++++++++++------- testkitbackend/_async/requests.py | 22 ++++++++++------- testkitbackend/_sync/requests.py | 22 ++++++++++------- 9 files changed, 99 insertions(+), 70 deletions(-) diff --git a/neo4j/_async/driver.py b/neo4j/_async/driver.py index 7bb16248e..b415cd4ed 100644 --- a/neo4j/_async/driver.py +++ b/neo4j/_async/driver.py @@ -389,6 +389,7 @@ def session( fetch_size: int = ..., impersonated_user: t.Optional[str] = ..., bookmarks: t.Union[t.Iterable[str], Bookmarks, None] = ..., + ignore_bookmark_manager: bool = ..., default_access_mode: str = ..., # undocumented/unsupported options diff --git a/neo4j/_async/work/session.py b/neo4j/_async/work/session.py index 81e5e58f1..67e526f38 100644 --- a/neo4j/_async/work/session.py +++ b/neo4j/_async/work/session.py @@ -102,9 +102,8 @@ class AsyncSession(AsyncWorkspace): def __init__(self, pool, session_config): assert isinstance(session_config, SessionConfig) super().__init__(pool, session_config) - if session_config.bookmarks is not None: - self._bookmarks = self._prepare_bookmarks(session_config.bookmarks) - else: + self._initialize_bookmarks(session_config.bookmarks) + if not session_config.ignore_bookmark_manager: self._bookmark_manager = session_config.bookmark_manager async def __aenter__(self) -> AsyncSession: @@ -119,21 +118,6 @@ async def __aexit__(self, exception_type, exception_value, traceback): self._state_failed = True await self.close() - def _prepare_bookmarks(self, bookmarks): - if isinstance(bookmarks, Bookmarks): - return tuple(bookmarks.raw_values) - if hasattr(bookmarks, "__iter__"): - deprecation_warn( - "Passing an iterable as `bookmarks` to `Session` is " - "deprecated. Please use a `Bookmarks` instance.", - stack_level=5 - ) - return tuple(bookmarks) - if not bookmarks: - return () - raise TypeError("Bookmarks must be an instance of Bookmarks or an " - "iterable of raw bookmarks (deprecated).") - async def _connect(self, access_mode, **access_kwargs): if access_mode is None: access_mode = self._config.default_access_mode diff --git a/neo4j/_async/work/workspace.py b/neo4j/_async/work/workspace.py index 37af39feb..b6506915e 100644 --- a/neo4j/_async/work/workspace.py +++ b/neo4j/_async/work/workspace.py @@ -27,6 +27,7 @@ deprecation_warn, unclosed_resource_warn, ) +from ...api import Bookmarks from ...exceptions import ( ServiceUnavailable, SessionError, @@ -46,6 +47,7 @@ def __init__(self, pool, config): # Sessions are supposed to cache the database on which to operate. self._cached_database = False self._bookmarks = () + self._initial_bookmarks = () self._bookmark_manager = None self._last_from_bookmark_manager = None # Workspace has been closed. @@ -80,13 +82,31 @@ def _set_cached_database(self, database): self._cached_database = True self._config.database = database + def _initialize_bookmarks(self, bookmarks): + if isinstance(bookmarks, Bookmarks): + prepared_bookmarks = tuple(bookmarks.raw_values) + elif hasattr(bookmarks, "__iter__"): + deprecation_warn( + "Passing an iterable as `bookmarks` to `Session` is " + "deprecated. Please use a `Bookmarks` instance.", + stack_level=5 + ) + prepared_bookmarks = tuple(bookmarks) + elif not bookmarks: + prepared_bookmarks = () + else: + raise TypeError("Bookmarks must be an instance of Bookmarks or an " + "iterable of raw bookmarks (deprecated).") + self._initial_bookmarks = self._bookmarks = prepared_bookmarks + async def _get_system_bookmarks(self): if self._bookmark_manager is not None: - self._bookmarks = tuple( - await AsyncUtil.callback( + self._bookmarks = tuple({ + *await AsyncUtil.callback( self._bookmark_manager.get_bookmarks, "system" - ) - ) + ), + *self._initial_bookmarks + }) return self._bookmarks async def _get_all_bookmarks(self): @@ -94,15 +114,19 @@ async def _get_all_bookmarks(self): if self._config.database: must_included_databases.append(self._config.database) if self._bookmark_manager is not None: - self._bookmarks = tuple( - await AsyncUtil.callback( + self._bookmarks = tuple({ + *await AsyncUtil.callback( self._bookmark_manager.get_all_bookmarks, must_included_databases - ) - ) + ), + *self._initial_bookmarks + }) return self._bookmarks async def _update_bookmarks(self, database, new_bookmarks): + if not new_bookmarks: + return + self._initial_bookmarks = () previous_bookmarks = self._bookmarks self._bookmarks = new_bookmarks if self._bookmark_manager is None: diff --git a/neo4j/_conf.py b/neo4j/_conf.py index a3e5a28a0..010452bce 100644 --- a/neo4j/_conf.py +++ b/neo4j/_conf.py @@ -444,6 +444,9 @@ class SessionConfig(WorkspaceConfig): #: Default AccessMode default_access_mode = WRITE_ACCESS + #: Whether to ignore the bookmark manager configured at driver level + ignore_bookmark_manager = False + class TransactionConfig(Config): """ Transaction configuration. This is internal for now. diff --git a/neo4j/_sync/driver.py b/neo4j/_sync/driver.py index a26e30349..781ca6c06 100644 --- a/neo4j/_sync/driver.py +++ b/neo4j/_sync/driver.py @@ -388,6 +388,7 @@ def session( fetch_size: int = ..., impersonated_user: t.Optional[str] = ..., bookmarks: t.Union[t.Iterable[str], Bookmarks, None] = ..., + ignore_bookmark_manager: bool = ..., default_access_mode: str = ..., # undocumented/unsupported options diff --git a/neo4j/_sync/work/session.py b/neo4j/_sync/work/session.py index 3fb01f86a..927db4e93 100644 --- a/neo4j/_sync/work/session.py +++ b/neo4j/_sync/work/session.py @@ -102,9 +102,8 @@ class Session(Workspace): def __init__(self, pool, session_config): assert isinstance(session_config, SessionConfig) super().__init__(pool, session_config) - if session_config.bookmarks is not None: - self._bookmarks = self._prepare_bookmarks(session_config.bookmarks) - else: + self._initialize_bookmarks(session_config.bookmarks) + if not session_config.ignore_bookmark_manager: self._bookmark_manager = session_config.bookmark_manager def __enter__(self) -> Session: @@ -119,21 +118,6 @@ def __exit__(self, exception_type, exception_value, traceback): self._state_failed = True self.close() - def _prepare_bookmarks(self, bookmarks): - if isinstance(bookmarks, Bookmarks): - return tuple(bookmarks.raw_values) - if hasattr(bookmarks, "__iter__"): - deprecation_warn( - "Passing an iterable as `bookmarks` to `Session` is " - "deprecated. Please use a `Bookmarks` instance.", - stack_level=5 - ) - return tuple(bookmarks) - if not bookmarks: - return () - raise TypeError("Bookmarks must be an instance of Bookmarks or an " - "iterable of raw bookmarks (deprecated).") - def _connect(self, access_mode, **access_kwargs): if access_mode is None: access_mode = self._config.default_access_mode diff --git a/neo4j/_sync/work/workspace.py b/neo4j/_sync/work/workspace.py index 4df469b80..d16f69579 100644 --- a/neo4j/_sync/work/workspace.py +++ b/neo4j/_sync/work/workspace.py @@ -27,6 +27,7 @@ deprecation_warn, unclosed_resource_warn, ) +from ...api import Bookmarks from ...exceptions import ( ServiceUnavailable, SessionError, @@ -46,6 +47,7 @@ def __init__(self, pool, config): # Sessions are supposed to cache the database on which to operate. self._cached_database = False self._bookmarks = () + self._initial_bookmarks = () self._bookmark_manager = None self._last_from_bookmark_manager = None # Workspace has been closed. @@ -80,13 +82,31 @@ def _set_cached_database(self, database): self._cached_database = True self._config.database = database + def _initialize_bookmarks(self, bookmarks): + if isinstance(bookmarks, Bookmarks): + prepared_bookmarks = tuple(bookmarks.raw_values) + elif hasattr(bookmarks, "__iter__"): + deprecation_warn( + "Passing an iterable as `bookmarks` to `Session` is " + "deprecated. Please use a `Bookmarks` instance.", + stack_level=5 + ) + prepared_bookmarks = tuple(bookmarks) + elif not bookmarks: + prepared_bookmarks = () + else: + raise TypeError("Bookmarks must be an instance of Bookmarks or an " + "iterable of raw bookmarks (deprecated).") + self._initial_bookmarks = self._bookmarks = prepared_bookmarks + def _get_system_bookmarks(self): if self._bookmark_manager is not None: - self._bookmarks = tuple( - Util.callback( + self._bookmarks = tuple({ + *Util.callback( self._bookmark_manager.get_bookmarks, "system" - ) - ) + ), + *self._initial_bookmarks + }) return self._bookmarks def _get_all_bookmarks(self): @@ -94,15 +114,19 @@ def _get_all_bookmarks(self): if self._config.database: must_included_databases.append(self._config.database) if self._bookmark_manager is not None: - self._bookmarks = tuple( - Util.callback( + self._bookmarks = tuple({ + *Util.callback( self._bookmark_manager.get_all_bookmarks, must_included_databases - ) - ) + ), + *self._initial_bookmarks + }) return self._bookmarks def _update_bookmarks(self, database, new_bookmarks): + if not new_bookmarks: + return + self._initial_bookmarks = () previous_bookmarks = self._bookmarks self._bookmarks = new_bookmarks if self._bookmark_manager is None: diff --git a/testkitbackend/_async/requests.py b/testkitbackend/_async/requests.py index 1c37e5fe3..81c2521b2 100644 --- a/testkitbackend/_async/requests.py +++ b/testkitbackend/_async/requests.py @@ -291,17 +291,21 @@ async def NewSession(backend, data): access_mode = neo4j.WRITE_ACCESS else: raise ValueError("Unknown access mode:" + access_mode) - bookmarks = None - if "bookmarks" in data and data["bookmarks"] is not None: - bookmarks = neo4j.Bookmarks.from_raw_values(data["bookmarks"]) config = { - "default_access_mode": access_mode, - "bookmarks": bookmarks, - "database": data["database"], - "fetch_size": data.get("fetchSize", None), - "impersonated_user": data.get("impersonatedUser", None), - + "default_access_mode": access_mode, + "database": data["database"], } + if data.get("bookmarks") is not None: + config["bookmarks"] = neo4j.Bookmarks.from_raw_values( + data["bookmarks"] + ) + for (conf_name, data_name) in ( + ("fetch_size", "fetchSize"), + ("impersonated_user", "impersonatedUser"), + ("ignore_bookmark_manager", "ignoreBookmarkManager"), + ): + if data_name in data: + config[conf_name] = data[data_name] session = driver.session(**config) key = backend.next_key() backend.sessions[key] = SessionTracker(session) diff --git a/testkitbackend/_sync/requests.py b/testkitbackend/_sync/requests.py index 4af03608f..2f77448f5 100644 --- a/testkitbackend/_sync/requests.py +++ b/testkitbackend/_sync/requests.py @@ -291,17 +291,21 @@ def NewSession(backend, data): access_mode = neo4j.WRITE_ACCESS else: raise ValueError("Unknown access mode:" + access_mode) - bookmarks = None - if "bookmarks" in data and data["bookmarks"] is not None: - bookmarks = neo4j.Bookmarks.from_raw_values(data["bookmarks"]) config = { - "default_access_mode": access_mode, - "bookmarks": bookmarks, - "database": data["database"], - "fetch_size": data.get("fetchSize", None), - "impersonated_user": data.get("impersonatedUser", None), - + "default_access_mode": access_mode, + "database": data["database"], } + if data.get("bookmarks") is not None: + config["bookmarks"] = neo4j.Bookmarks.from_raw_values( + data["bookmarks"] + ) + for (conf_name, data_name) in ( + ("fetch_size", "fetchSize"), + ("impersonated_user", "impersonatedUser"), + ("ignore_bookmark_manager", "ignoreBookmarkManager"), + ): + if data_name in data: + config[conf_name] = data[data_name] session = driver.session(**config) key = backend.next_key() backend.sessions[key] = SessionTracker(session) From 693b9b51cace24d814ce54b09527ec09eaac8c01 Mon Sep 17 00:00:00 2001 From: Rouven Bauer Date: Mon, 8 Aug 2022 16:50:53 +0200 Subject: [PATCH 06/26] Lock bookmark manager per db --- neo4j/_async/bookmark_manager.py | 51 ++++++++++++++------------------ neo4j/_sync/bookmark_manager.py | 51 ++++++++++++++------------------ 2 files changed, 44 insertions(+), 58 deletions(-) diff --git a/neo4j/_async/bookmark_manager.py b/neo4j/_async/bookmark_manager.py index c068595e7..4ee46667d 100644 --- a/neo4j/_async/bookmark_manager.py +++ b/neo4j/_async/bookmark_manager.py @@ -21,10 +21,7 @@ import typing as t from collections import defaultdict -from .._async_compat.concurrency import ( - AsyncCooperativeLock, - AsyncLock, -) +from .._async_compat.concurrency import AsyncCooperativeLock from .._async_compat.util import AsyncUtil from ..api import AsyncBookmarkManager @@ -40,51 +37,47 @@ def __init__(self, initial_bookmarks=None, bookmark_supplier=None, self._bookmarks = defaultdict( set, ((k, set(v)) for k, v in initial_bookmarks.items()) ) - self._lock: t.Union[AsyncLock, AsyncCooperativeLock] - if bookmark_supplier or notify_bookmarks: - self._lock = AsyncLock() - else: - self._lock = AsyncCooperativeLock() + # the value of self._bookmarks[db] may only be changed with + # self._per_db_lock[db] acquired + self._per_db_lock = defaultdict(AsyncCooperativeLock) async def update_bookmarks( self, database: str, previous_bookmarks: t.Iterable[str], new_bookmarks: t.Iterable[str] ) -> None: - async with self._lock: - new_bms = set(new_bookmarks) + new_bms = set(new_bookmarks) + prev_bms = set(previous_bookmarks) + with self._per_db_lock[database]: if not new_bms: return - prev_bms = set(previous_bookmarks) curr_bms = self._bookmarks[database] curr_bms.difference_update(prev_bms) curr_bms.update(new_bms) - if self._notify_bookmarks: - await AsyncUtil.callback( - self._notify_bookmarks, database, tuple(curr_bms) - ) + curr_bms_snapshot = tuple(curr_bms) + if self._notify_bookmarks: + await AsyncUtil.callback(self._notify_bookmarks, + database, curr_bms_snapshot) async def _get_bookmarks(self, database: str) -> t.Set[str]: - bms = self._bookmarks[database] + with self._per_db_lock[database]: + bms = set(self._bookmarks[database]) if self._bookmark_supplier: - extra_bms = await AsyncUtil.callback( - self._bookmark_supplier, database - ) + extra_bms = await AsyncUtil.callback(self._bookmark_supplier, + database) if extra_bms is not None: bms &= set(extra_bms) return bms async def get_bookmarks(self, database: str) -> t.Set[str]: - async with self._lock: - return await self._get_bookmarks(database) + return await self._get_bookmarks(database) async def get_all_bookmarks( self, must_included_databases: t.Iterable[str] ) -> t.Set[str]: - async with self._lock: - bms = set() - databases = (set(must_included_databases) - | set(self._bookmarks.keys())) - for database in databases: - bms.update(await self._get_bookmarks(database)) - return bms + bms = set() + databases = (set(must_included_databases) + | set(self._bookmarks.keys())) + for database in databases: + bms.update(await self._get_bookmarks(database)) + return bms diff --git a/neo4j/_sync/bookmark_manager.py b/neo4j/_sync/bookmark_manager.py index 5b7560e1a..ac4d58ce3 100644 --- a/neo4j/_sync/bookmark_manager.py +++ b/neo4j/_sync/bookmark_manager.py @@ -21,10 +21,7 @@ import typing as t from collections import defaultdict -from .._async_compat.concurrency import ( - CooperativeLock, - Lock, -) +from .._async_compat.concurrency import CooperativeLock from .._async_compat.util import Util from ..api import BookmarkManager @@ -40,51 +37,47 @@ def __init__(self, initial_bookmarks=None, bookmark_supplier=None, self._bookmarks = defaultdict( set, ((k, set(v)) for k, v in initial_bookmarks.items()) ) - self._lock: t.Union[Lock, CooperativeLock] - if bookmark_supplier or notify_bookmarks: - self._lock = Lock() - else: - self._lock = CooperativeLock() + # the value of self._bookmarks[db] may only be changed with + # self._per_db_lock[db] acquired + self._per_db_lock = defaultdict(CooperativeLock) def update_bookmarks( self, database: str, previous_bookmarks: t.Iterable[str], new_bookmarks: t.Iterable[str] ) -> None: - with self._lock: - new_bms = set(new_bookmarks) + new_bms = set(new_bookmarks) + prev_bms = set(previous_bookmarks) + with self._per_db_lock[database]: if not new_bms: return - prev_bms = set(previous_bookmarks) curr_bms = self._bookmarks[database] curr_bms.difference_update(prev_bms) curr_bms.update(new_bms) - if self._notify_bookmarks: - Util.callback( - self._notify_bookmarks, database, tuple(curr_bms) - ) + curr_bms_snapshot = tuple(curr_bms) + if self._notify_bookmarks: + Util.callback(self._notify_bookmarks, + database, curr_bms_snapshot) def _get_bookmarks(self, database: str) -> t.Set[str]: - bms = self._bookmarks[database] + with self._per_db_lock[database]: + bms = set(self._bookmarks[database]) if self._bookmark_supplier: - extra_bms = Util.callback( - self._bookmark_supplier, database - ) + extra_bms = Util.callback(self._bookmark_supplier, + database) if extra_bms is not None: bms &= set(extra_bms) return bms def get_bookmarks(self, database: str) -> t.Set[str]: - with self._lock: - return self._get_bookmarks(database) + return self._get_bookmarks(database) def get_all_bookmarks( self, must_included_databases: t.Iterable[str] ) -> t.Set[str]: - with self._lock: - bms = set() - databases = (set(must_included_databases) - | set(self._bookmarks.keys())) - for database in databases: - bms.update(self._get_bookmarks(database)) - return bms + bms = set() + databases = (set(must_included_databases) + | set(self._bookmarks.keys())) + for database in databases: + bms.update(self._get_bookmarks(database)) + return bms From 4ec20b35242bf751c3311007241125a2770e9cf8 Mon Sep 17 00:00:00 2001 From: Rouven Bauer Date: Mon, 8 Aug 2022 16:51:30 +0200 Subject: [PATCH 07/26] Add `forget` method to bookmark manager --- neo4j/_async/bookmark_manager.py | 5 +++++ neo4j/_sync/bookmark_manager.py | 5 +++++ neo4j/api.py | 16 ++++++++++++++++ 3 files changed, 26 insertions(+) diff --git a/neo4j/_async/bookmark_manager.py b/neo4j/_async/bookmark_manager.py index 4ee46667d..e2b2688b2 100644 --- a/neo4j/_async/bookmark_manager.py +++ b/neo4j/_async/bookmark_manager.py @@ -81,3 +81,8 @@ async def get_all_bookmarks( for database in databases: bms.update(await self._get_bookmarks(database)) return bms + + async def forget(self, databases: t.Iterable[str]) -> None: + for database in databases: + with self._per_db_lock[database]: + del self._bookmarks[database] diff --git a/neo4j/_sync/bookmark_manager.py b/neo4j/_sync/bookmark_manager.py index ac4d58ce3..fb0b98b9d 100644 --- a/neo4j/_sync/bookmark_manager.py +++ b/neo4j/_sync/bookmark_manager.py @@ -81,3 +81,8 @@ def get_all_bookmarks( for database in databases: bms.update(self._get_bookmarks(database)) return bms + + def forget(self, databases: t.Iterable[str]) -> None: + for database in databases: + with self._per_db_lock[database]: + del self._bookmarks[database] diff --git a/neo4j/api.py b/neo4j/api.py index a11eec821..cd822f4d2 100644 --- a/neo4j/api.py +++ b/neo4j/api.py @@ -442,6 +442,18 @@ def get_all_bookmarks( """ ... + @abc.abstractmethod + def forget(self, databases: t.Iterable[str]) -> None: + """Forget the bookmarks for the given databases. + + This method is not called by the driver. + Forgetting unused databases is the user's responsibility. + + :param databases: + The databases which the bookmarks will be removed for. + """ + ... + class AsyncBookmarkManager(abc.ABC): """Same as :class:`BookmarkManager` but with async methods.""" @@ -463,6 +475,10 @@ async def get_all_bookmarks( ) -> t.Collection[str]: ... + @abc.abstractmethod + async def forget(self, databases: t.Iterable[str]) -> None: + ... + def parse_neo4j_uri(uri): parsed = urlparse(uri) From ad092c9ded89cd7838f4555285fe59305c47d99e Mon Sep 17 00:00:00 2001 From: Rouven Bauer Date: Thu, 11 Aug 2022 14:06:18 +0200 Subject: [PATCH 08/26] Add API docs for the bookmark manager --- docs/source/api.rst | 64 +++++++++++++++++++--------- docs/source/async_api.rst | 87 +++++++++++++++++++++++++-------------- neo4j/_async/driver.py | 20 ++++++++- neo4j/_sync/driver.py | 18 +++++++- neo4j/api.py | 24 ++++++++++- 5 files changed, 159 insertions(+), 54 deletions(-) diff --git a/docs/source/api.rst b/docs/source/api.rst index fadef6a3f..4c9b832db 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -14,40 +14,41 @@ Driver Construction The :class:`neo4j.Driver` construction is done via a ``classmethod`` on the :class:`neo4j.GraphDatabase` class. .. autoclass:: neo4j.GraphDatabase - :members: driver + :members: bookmark_manager + .. method:: driver -Driver creation example: + Driver creation example: -.. code-block:: python + .. code-block:: python - from neo4j import GraphDatabase + from neo4j import GraphDatabase - uri = "neo4j://example.com:7687" - driver = GraphDatabase.driver(uri, auth=("neo4j", "password")) + uri = "neo4j://example.com:7687" + driver = GraphDatabase.driver(uri, auth=("neo4j", "password")) - driver.close() # close the driver object + driver.close() # close the driver object -For basic authentication, ``auth`` can be a simple tuple, for example: + For basic authentication, ``auth`` can be a simple tuple, for example: -.. code-block:: python + .. code-block:: python - auth = ("neo4j", "password") + auth = ("neo4j", "password") -This will implicitly create a :class:`neo4j.Auth` with a ``scheme="basic"``. -Other authentication methods are described under :ref:`auth-ref`. + This will implicitly create a :class:`neo4j.Auth` with a ``scheme="basic"``. + Other authentication methods are described under :ref:`auth-ref`. -``with`` block context example: + ``with`` block context example: -.. code-block:: python + .. code-block:: python - from neo4j import GraphDatabase + from neo4j import GraphDatabase - uri = "neo4j://example.com:7687" - with GraphDatabase.driver(uri, auth=("neo4j", "password")) as driver: - # use the driver + uri = "neo4j://example.com:7687" + with GraphDatabase.driver(uri, auth=("neo4j", "password")) as driver: + # use the driver @@ -138,7 +139,7 @@ Alternatively, one of the auth token helper functions can be used. Driver ****** -Every Neo4j-backed application will require a :class:`neo4j.Driver` object. +Every Neo4j-backed application will require a driver object. This object holds the details required to establish connections with a Neo4j database, including server URIs, credentials and other configuration. :class:`neo4j.Driver` objects hold a connection pool from which :class:`neo4j.Session` objects can borrow connections. @@ -174,6 +175,7 @@ Additional configuration can be provided via the :class:`neo4j.Driver` construct + :ref:`ssl-context-ref` + :ref:`trusted-certificates-ref` + :ref:`user-agent-ref` ++ :ref:`bookmark-manager-ref` .. _session-connection-timeout-ref: @@ -417,6 +419,21 @@ Specify the client agent name. :Default: *The Python Driver will generate a user agent name.* +.. _bookmark-manager-ref: + +``bookmark_manager`` +-------------------- +Specify a bookmark manager for the driver to use. If present, the bookmark +manger is used to keep all work on the driver causally consistent. + +See :class:`.BookmarkManager` for more information. + +:Type: :const:`None` or :class:`.BookmarkManager` +:Default: :const:`None` + +.. versionadded:: 5.0 + + Driver Object Lifetime ====================== @@ -1227,6 +1244,15 @@ Temporal Data Types See topic :ref:`temporal-data-types` for more details. +*************** +BookmarkManager +*************** + +.. autoclass:: neo4j.api.BookmarkManager + :show-inheritance: + :members: + + .. _errors-ref: ****** diff --git a/docs/source/async_api.rst b/docs/source/async_api.rst index 61f2bb2c3..eefe1e6fa 100644 --- a/docs/source/async_api.rst +++ b/docs/source/async_api.rst @@ -29,51 +29,53 @@ Async Driver Construction The :class:`neo4j.AsyncDriver` construction is done via a ``classmethod`` on the :class:`neo4j.AsyncGraphDatabase` class. .. autoclass:: neo4j.AsyncGraphDatabase - :members: driver + :members: bookmark_manager + .. automethod:: driver -Driver creation example: + Driver creation example: -.. code-block:: python + .. code-block:: python - import asyncio + import asyncio - from neo4j import AsyncGraphDatabase + from neo4j import AsyncGraphDatabase - async def main(): - uri = "neo4j://example.com:7687" - driver = AsyncGraphDatabase.driver(uri, auth=("neo4j", "password")) + async def main(): + uri = "neo4j://example.com:7687" + driver = AsyncGraphDatabase.driver(uri, auth=("neo4j", "password")) - await driver.close() # close the driver object + await driver.close() # close the driver object - asyncio.run(main()) + asyncio.run(main()) -For basic authentication, ``auth`` can be a simple tuple, for example: + For basic authentication, ``auth`` can be a simple tuple, for example: -.. code-block:: python + .. code-block:: python - auth = ("neo4j", "password") + auth = ("neo4j", "password") -This will implicitly create a :class:`neo4j.Auth` with a ``scheme="basic"``. -Other authentication methods are described under :ref:`auth-ref`. + This will implicitly create a :class:`neo4j.Auth` with a ``scheme="basic"``. + Other authentication methods are described under :ref:`auth-ref`. -``with`` block context example: + ``with`` block context example: -.. code-block:: python + .. code-block:: python - import asyncio + import asyncio - from neo4j import AsyncGraphDatabase + from neo4j import AsyncGraphDatabase - async def main(): - uri = "neo4j://example.com:7687" - auth = ("neo4j", "password") - async with AsyncGraphDatabase.driver(uri, auth=auth) as driver: - # use the driver - ... + async def main(): + uri = "neo4j://example.com:7687" + auth = ("neo4j", "password") + async with AsyncGraphDatabase.driver(uri, auth=auth) as driver: + # use the driver + ... + + asyncio.run(main()) - asyncio.run(main()) .. _async-uri-ref: @@ -128,7 +130,7 @@ Each supported scheme maps to a particular :class:`neo4j.AsyncDriver` subclass t AsyncDriver *********** -Every Neo4j-backed application will require a :class:`neo4j.AsyncDriver` object. +Every Neo4j-backed application will require a driver object. This object holds the details required to establish connections with a Neo4j database, including server URIs, credentials and other configuration. :class:`neo4j.AsyncDriver` objects hold a connection pool from which :class:`neo4j.AsyncSession` objects can borrow connections. @@ -149,8 +151,13 @@ Async Driver Configuration ========================== :class:`neo4j.AsyncDriver` is configured exactly like :class:`neo4j.Driver` -(see :ref:`driver-configuration-ref`). The only difference is that the async -driver accepts an async custom resolver function: +(see :ref:`driver-configuration-ref`). The only differences are: +* the async driver accepts an async custom resolver function, +see :ref:`async-resolver-ref`. +* the async driver accepts accepts either a :class:`neo4j.api.BookmarkManager` +object or a :class:`neo4j.api.AsyncBookmarkManager` as bookmark manager. +see :ref:`async-bookmark-manager-ref`. + .. _async-resolver-ref: @@ -189,6 +196,19 @@ For example: :Default: ``None`` +.. _async-bookmark-manager-ref: + +``bookmark_manager`` +-------------------- +Specify a bookmark manager for the driver to use. If present, the bookmark +manger is used to keep all work on the driver causally consistent. + +See :class:`BookmarkManager` for more information. + +:Type: :const:`None`, :class:`BookmarkManager`, or :class:`AsyncBookmarkManager` +:Default: ``None`` + + Driver Object Lifetime ====================== @@ -526,7 +546,6 @@ Example: To exert more control over how a transaction function is carried out, the :func:`neo4j.unit_of_work` decorator can be used. - *********** AsyncResult *********** @@ -572,6 +591,14 @@ A :class:`neo4j.AsyncResult` is attached to an active connection, through a :cla See https://neo4j.com/docs/python-manual/current/cypher-workflow/#python-driver-type-mapping for more about type mapping. +******************** +AsyncBookmarkManager +******************** + +.. autoclass:: neo4j.api.AsyncBookmarkManager + :show-inheritance: + :members: + ****************** Async Cancellation diff --git a/neo4j/_async/driver.py b/neo4j/_async/driver.py index b415cd4ed..6d2a1f191 100644 --- a/neo4j/_async/driver.py +++ b/neo4j/_async/driver.py @@ -73,7 +73,7 @@ class AsyncGraphDatabase: - """Accessor for :class:`neo4j.Driver` construction. + """Accessor for :class:`neo4j.AsyncDriver` construction. """ if t.TYPE_CHECKING: @@ -225,7 +225,21 @@ def bookmark_manager( bookmark_supplier: _T_BmSupplier = None, notify_bookmarks: _T_NotifyBm = None ) -> AsyncBookmarkManager: - """Create a default :class:`AsyncBookmarkManager`. + """Create a default :class:`.AsyncBookmarkManager`. + + Basic usage example to configure the driver with the default + bookmark manger implementation so that all work is automatically + causally chained (i.e., all reads can observe all previous writes + even in a clustered setup):: + + import neo4j + + driver = neo4j.AsyncGraphDatabase.driver( + uri, auth=..., # ... + bookmark_manager=neo4j.AsyncGraphDatabase.bookmark_manager( + # ... configure the bookmark manager + ) + ) :param initial_bookmarks: The initial set of bookmarks. The default bookmark manager will @@ -244,6 +258,8 @@ def bookmark_manager( internal bookmark set. :returns: A default implementation of :class:`AsyncBookmarkManager`. + + .. versionadded:: 5.0 """ return AsyncNeo4jBookmarkManager( initial_bookmarks=initial_bookmarks, diff --git a/neo4j/_sync/driver.py b/neo4j/_sync/driver.py index 781ca6c06..3d76ce8db 100644 --- a/neo4j/_sync/driver.py +++ b/neo4j/_sync/driver.py @@ -224,7 +224,21 @@ def bookmark_manager( bookmark_supplier: _T_BmSupplier = None, notify_bookmarks: _T_NotifyBm = None ) -> BookmarkManager: - """Create a default :class:`BookmarkManager`. + """Create a default :class:`.BookmarkManager`. + + Basic usage example to configure the driver with the default + bookmark manger implementation so that all work is automatically + causally chained (i.e., all reads can observe all previous writes + even in a clustered setup):: + + import neo4j + + driver = neo4j.GraphDatabase.driver( + uri, auth=..., # ... + bookmark_manager=neo4j.GraphDatabase.bookmark_manager( + # ... configure the bookmark manager + ) + ) :param initial_bookmarks: The initial set of bookmarks. The default bookmark manager will @@ -243,6 +257,8 @@ def bookmark_manager( internal bookmark set. :returns: A default implementation of :class:`BookmarkManager`. + + .. versionadded:: 5.0 """ return Neo4jBookmarkManager( initial_bookmarks=initial_bookmarks, diff --git a/neo4j/api.py b/neo4j/api.py index cd822f4d2..c90f36bc4 100644 --- a/neo4j/api.py +++ b/neo4j/api.py @@ -391,10 +391,16 @@ class BookmarkManager(abc.ABC): The bookmark manager is an interface used by the driver for keeping track of the bookmarks and this way keeping sessions automatically - consistent. + consistent. Configure the driver to use a specific bookmark manager with + :ref:`bookmark-manager-ref`. + + The driver comes with a default implementation of the bookmark manager + accessible through :attr:`.GraphDatabase.bookmark_manager()`. .. note:: All methods must be concurrency safe. + + .. versionadded:: 5.0 """ @abc.abstractmethod @@ -456,7 +462,13 @@ def forget(self, databases: t.Iterable[str]) -> None: class AsyncBookmarkManager(abc.ABC): - """Same as :class:`BookmarkManager` but with async methods.""" + """Same as :class:`.BookmarkManager` but with async methods. + + The driver comes with a default implementation of the async bookmark + manager accessible through :attr:`.AsyncGraphDatabase.bookmark_manager()`. + + .. versionadded:: 5.0 + """ @abc.abstractmethod async def update_bookmarks( @@ -465,20 +477,28 @@ async def update_bookmarks( ) -> None: ... + update_bookmarks.__doc__ = BookmarkManager.update_bookmarks.__doc__ + @abc.abstractmethod async def get_bookmarks(self, database: str) -> t.Collection[str]: ... + get_bookmarks.__doc__ = BookmarkManager.get_bookmarks.__doc__ + @abc.abstractmethod async def get_all_bookmarks( self, must_included_databases: t.Iterable[str] ) -> t.Collection[str]: ... + get_all_bookmarks.__doc__ = BookmarkManager.get_all_bookmarks.__doc__ + @abc.abstractmethod async def forget(self, databases: t.Iterable[str]) -> None: ... + forget.__doc__ = BookmarkManager.forget.__doc__ + def parse_neo4j_uri(uri): parsed = urlparse(uri) From e804076fd26acb3fe629b97566b9666441a6df2b Mon Sep 17 00:00:00 2001 From: Rouven Bauer Date: Thu, 11 Aug 2022 16:46:49 +0200 Subject: [PATCH 09/26] Implement bookmark_consumer and remove must_included_databases Implement ADR adjustments to bookmark manager https://github.com/neo-technology/drivers-adr/pull/30/ --- neo4j/_async/bookmark_manager.py | 75 ++++++++++++++++++-------------- neo4j/_async/driver.py | 29 +++++++----- neo4j/_async_compat/util.py | 38 ++++++++++++++++ neo4j/_sync/bookmark_manager.py | 75 ++++++++++++++++++-------------- neo4j/_sync/driver.py | 29 +++++++----- neo4j/api.py | 19 ++------ 6 files changed, 161 insertions(+), 104 deletions(-) diff --git a/neo4j/_async/bookmark_manager.py b/neo4j/_async/bookmark_manager.py index e2b2688b2..4ffab861c 100644 --- a/neo4j/_async/bookmark_manager.py +++ b/neo4j/_async/bookmark_manager.py @@ -23,23 +23,34 @@ from .._async_compat.concurrency import AsyncCooperativeLock from .._async_compat.util import AsyncUtil -from ..api import AsyncBookmarkManager +from ..api import ( + AsyncBookmarkManager, + Bookmarks, +) + + +T_BmSupplier = t.Callable[[t.Optional[str]], + t.Union[Bookmarks, t.Awaitable[Bookmarks]]] +T_BmConsumer = t.Callable[[str, Bookmarks], t.Union[None, t.Awaitable[None]]] class AsyncNeo4jBookmarkManager(AsyncBookmarkManager): - def __init__(self, initial_bookmarks=None, bookmark_supplier=None, - notify_bookmarks=None): + def __init__( + self, + initial_bookmarks: t.Mapping[str, t.Iterable[str]] = None, + bookmark_supplier: T_BmSupplier = None, + bookmarks_consumer: T_BmConsumer = None + ) -> None: super().__init__() self._bookmark_supplier = bookmark_supplier - self._notify_bookmarks = notify_bookmarks + self._bookmarks_consumer = bookmarks_consumer if initial_bookmarks is None: initial_bookmarks = {} self._bookmarks = defaultdict( - set, ((k, set(v)) for k, v in initial_bookmarks.items()) + set, ((k, set(map(str, v))) + for k, v in initial_bookmarks.items()) ) - # the value of self._bookmarks[db] may only be changed with - # self._per_db_lock[db] acquired - self._per_db_lock = defaultdict(AsyncCooperativeLock) + self._lock = AsyncCooperativeLock() async def update_bookmarks( self, database: str, previous_bookmarks: t.Iterable[str], @@ -47,42 +58,42 @@ async def update_bookmarks( ) -> None: new_bms = set(new_bookmarks) prev_bms = set(previous_bookmarks) - with self._per_db_lock[database]: + with self._lock: if not new_bms: return curr_bms = self._bookmarks[database] curr_bms.difference_update(prev_bms) curr_bms.update(new_bms) - if self._notify_bookmarks: - curr_bms_snapshot = tuple(curr_bms) - if self._notify_bookmarks: - await AsyncUtil.callback(self._notify_bookmarks, - database, curr_bms_snapshot) + if self._bookmarks_consumer: + curr_bms_snapshot = Bookmarks.from_raw_values(curr_bms) + if self._bookmarks_consumer: + await AsyncUtil.callback( + self._bookmarks_consumer, database, curr_bms_snapshot + ) - async def _get_bookmarks(self, database: str) -> t.Set[str]: - with self._per_db_lock[database]: + async def get_bookmarks(self, database: str) -> t.Set[str]: + with self._lock: bms = set(self._bookmarks[database]) if self._bookmark_supplier: - extra_bms = await AsyncUtil.callback(self._bookmark_supplier, - database) - if extra_bms is not None: - bms &= set(extra_bms) + extra_bms = await AsyncUtil.callback( + self._bookmark_supplier, database + ) + bms.update(extra_bms) return bms - async def get_bookmarks(self, database: str) -> t.Set[str]: - return await self._get_bookmarks(database) - - async def get_all_bookmarks( - self, must_included_databases: t.Iterable[str] - ) -> t.Set[str]: - bms = set() - databases = (set(must_included_databases) - | set(self._bookmarks.keys())) - for database in databases: - bms.update(await self._get_bookmarks(database)) + async def get_all_bookmarks(self) -> t.Set[str]: + bms: t.Set[str] = set() + with self._lock: + for database in self._bookmarks.keys(): + bms.update(self._bookmarks[database]) + if self._bookmark_supplier: + extra_bms = await AsyncUtil.callback( + self._bookmark_supplier, None + ) + bms.update(extra_bms) return bms async def forget(self, databases: t.Iterable[str]) -> None: for database in databases: - with self._per_db_lock[database]: + with self._lock: del self._bookmarks[database] diff --git a/neo4j/_async/driver.py b/neo4j/_async/driver.py index 6d2a1f191..fa90e4e1d 100644 --- a/neo4j/_async/driver.py +++ b/neo4j/_async/driver.py @@ -64,14 +64,14 @@ URI_SCHEME_NEO4J_SECURE, URI_SCHEME_NEO4J_SELF_SIGNED_CERTIFICATE, ) -from .bookmark_manager import AsyncNeo4jBookmarkManager +from .bookmark_manager import ( + AsyncNeo4jBookmarkManager, + T_BmConsumer as _T_BmConsumer, + T_BmSupplier as _T_BmSupplier, +) from .work import AsyncSession -_T_BmSupplier = t.Callable[[str], t.Union[Bookmarks, t.Awaitable[Bookmarks]]] -_T_NotifyBm = t.Callable[[str, Bookmarks], t.Union[None, t.Awaitable[None]]] - - class AsyncGraphDatabase: """Accessor for :class:`neo4j.AsyncDriver` construction. """ @@ -223,7 +223,7 @@ def bookmark_manager( initial_bookmarks: t.Mapping[str, t.Union[Bookmarks, t.Iterable[str]]] = None, bookmark_supplier: _T_BmSupplier = None, - notify_bookmarks: _T_NotifyBm = None + bookmarks_consumer: _T_BmConsumer = None ) -> AsyncBookmarkManager: """Create a default :class:`.AsyncBookmarkManager`. @@ -249,13 +249,18 @@ def bookmark_manager( :param bookmark_supplier: Function which will be called every time the default bookmark manager's method :meth:`.AsyncBookmarkManager.get_bookmarks` - gets called. The result of ``bookmark_supplier`` will be - concatenated with the internal set of bookmarks and used to - configure the session in creation. - :param notify_bookmarks: + or :meth:`.AsyncBookmarkManager.get_all_bookmarks` gets called. + The function will be passed the name of the database (``str``) if + ``.get_bookmarks`` is called or ``None`` if ``.get_all_bookmarks`` + is called. The function must return a :class:`.Bookmarks` object. + The result of ``bookmark_supplier`` will then be concatenated with + the internal set of bookmarks and used to configure the session in + creation. + :param bookmarks_consumer: Function which will be called whenever the set of bookmarks handled by the bookmark manager gets updated with the new - internal bookmark set. + internal bookmark set. It will receive the name of the database + and the new set of bookmarks. :returns: A default implementation of :class:`AsyncBookmarkManager`. @@ -264,7 +269,7 @@ def bookmark_manager( return AsyncNeo4jBookmarkManager( initial_bookmarks=initial_bookmarks, bookmark_supplier=bookmark_supplier, - notify_bookmarks=notify_bookmarks + bookmarks_consumer=bookmarks_consumer ) @classmethod diff --git a/neo4j/_async_compat/util.py b/neo4j/_async_compat/util.py index f8c116556..c7ed041ed 100644 --- a/neo4j/_async_compat/util.py +++ b/neo4j/_async_compat/util.py @@ -16,13 +16,23 @@ # limitations under the License. +from __future__ import annotations + import asyncio import inspect +import typing as t from functools import wraps from .._meta import experimental +if t.TYPE_CHECKING: + import typing_extensions as te + + _T = t.TypeVar("_T") + _P = te.ParamSpec("_P") + + __all__ = [ "AsyncUtil", "Util", @@ -43,6 +53,23 @@ async def next(it): async def list(it): return [x async for x in it] + @staticmethod + @t.overload + async def callback(cb: None, *args: object, **kwargs: object) -> None: + ... + + @staticmethod + @t.overload + async def callback( + cb: t.Union[ + t.Callable[_P, t.Union[_T, t.Awaitable[_T]]], + t.Callable[_P, t.Awaitable[_T]], + t.Callable[_P, _T], + ], + *args: _P.args, **kwargs: _P.kwargs + ) -> _T: + ... + @staticmethod async def callback(cb, *args, **kwargs): if callable(cb): @@ -71,6 +98,17 @@ class Util: next = next list = list + @staticmethod + @t.overload + def callback(cb: None, *args: object, **kwargs: object) -> None: + ... + + @staticmethod + @t.overload + def callback(cb: t.Callable[_P, _T], + *args: _P.args, **kwargs: _P.kwargs) -> _T: + ... + @staticmethod def callback(cb, *args, **kwargs): if callable(cb): diff --git a/neo4j/_sync/bookmark_manager.py b/neo4j/_sync/bookmark_manager.py index fb0b98b9d..4a46614a0 100644 --- a/neo4j/_sync/bookmark_manager.py +++ b/neo4j/_sync/bookmark_manager.py @@ -23,23 +23,34 @@ from .._async_compat.concurrency import CooperativeLock from .._async_compat.util import Util -from ..api import BookmarkManager +from ..api import ( + BookmarkManager, + Bookmarks, +) + + +T_BmSupplier = t.Callable[[t.Optional[str]], + t.Union[Bookmarks, t.Union[Bookmarks]]] +T_BmConsumer = t.Callable[[str, Bookmarks], t.Union[None, t.Union[None]]] class Neo4jBookmarkManager(BookmarkManager): - def __init__(self, initial_bookmarks=None, bookmark_supplier=None, - notify_bookmarks=None): + def __init__( + self, + initial_bookmarks: t.Mapping[str, t.Iterable[str]] = None, + bookmark_supplier: T_BmSupplier = None, + bookmarks_consumer: T_BmConsumer = None + ) -> None: super().__init__() self._bookmark_supplier = bookmark_supplier - self._notify_bookmarks = notify_bookmarks + self._bookmarks_consumer = bookmarks_consumer if initial_bookmarks is None: initial_bookmarks = {} self._bookmarks = defaultdict( - set, ((k, set(v)) for k, v in initial_bookmarks.items()) + set, ((k, set(map(str, v))) + for k, v in initial_bookmarks.items()) ) - # the value of self._bookmarks[db] may only be changed with - # self._per_db_lock[db] acquired - self._per_db_lock = defaultdict(CooperativeLock) + self._lock = CooperativeLock() def update_bookmarks( self, database: str, previous_bookmarks: t.Iterable[str], @@ -47,42 +58,42 @@ def update_bookmarks( ) -> None: new_bms = set(new_bookmarks) prev_bms = set(previous_bookmarks) - with self._per_db_lock[database]: + with self._lock: if not new_bms: return curr_bms = self._bookmarks[database] curr_bms.difference_update(prev_bms) curr_bms.update(new_bms) - if self._notify_bookmarks: - curr_bms_snapshot = tuple(curr_bms) - if self._notify_bookmarks: - Util.callback(self._notify_bookmarks, - database, curr_bms_snapshot) + if self._bookmarks_consumer: + curr_bms_snapshot = Bookmarks.from_raw_values(curr_bms) + if self._bookmarks_consumer: + Util.callback( + self._bookmarks_consumer, database, curr_bms_snapshot + ) - def _get_bookmarks(self, database: str) -> t.Set[str]: - with self._per_db_lock[database]: + def get_bookmarks(self, database: str) -> t.Set[str]: + with self._lock: bms = set(self._bookmarks[database]) if self._bookmark_supplier: - extra_bms = Util.callback(self._bookmark_supplier, - database) - if extra_bms is not None: - bms &= set(extra_bms) + extra_bms = Util.callback( + self._bookmark_supplier, database + ) + bms.update(extra_bms) return bms - def get_bookmarks(self, database: str) -> t.Set[str]: - return self._get_bookmarks(database) - - def get_all_bookmarks( - self, must_included_databases: t.Iterable[str] - ) -> t.Set[str]: - bms = set() - databases = (set(must_included_databases) - | set(self._bookmarks.keys())) - for database in databases: - bms.update(self._get_bookmarks(database)) + def get_all_bookmarks(self) -> t.Set[str]: + bms: t.Set[str] = set() + with self._lock: + for database in self._bookmarks.keys(): + bms.update(self._bookmarks[database]) + if self._bookmark_supplier: + extra_bms = Util.callback( + self._bookmark_supplier, None + ) + bms.update(extra_bms) return bms def forget(self, databases: t.Iterable[str]) -> None: for database in databases: - with self._per_db_lock[database]: + with self._lock: del self._bookmarks[database] diff --git a/neo4j/_sync/driver.py b/neo4j/_sync/driver.py index 3d76ce8db..595a7d991 100644 --- a/neo4j/_sync/driver.py +++ b/neo4j/_sync/driver.py @@ -63,14 +63,14 @@ URI_SCHEME_NEO4J_SECURE, URI_SCHEME_NEO4J_SELF_SIGNED_CERTIFICATE, ) -from .bookmark_manager import Neo4jBookmarkManager +from .bookmark_manager import ( + Neo4jBookmarkManager, + T_BmConsumer as _T_BmConsumer, + T_BmSupplier as _T_BmSupplier, +) from .work import Session -_T_BmSupplier = t.Callable[[str], t.Union[Bookmarks, t.Union[Bookmarks]]] -_T_NotifyBm = t.Callable[[str, Bookmarks], t.Union[None, t.Union[None]]] - - class GraphDatabase: """Accessor for :class:`neo4j.Driver` construction. """ @@ -222,7 +222,7 @@ def bookmark_manager( initial_bookmarks: t.Mapping[str, t.Union[Bookmarks, t.Iterable[str]]] = None, bookmark_supplier: _T_BmSupplier = None, - notify_bookmarks: _T_NotifyBm = None + bookmarks_consumer: _T_BmConsumer = None ) -> BookmarkManager: """Create a default :class:`.BookmarkManager`. @@ -248,13 +248,18 @@ def bookmark_manager( :param bookmark_supplier: Function which will be called every time the default bookmark manager's method :meth:`.BookmarkManager.get_bookmarks` - gets called. The result of ``bookmark_supplier`` will be - concatenated with the internal set of bookmarks and used to - configure the session in creation. - :param notify_bookmarks: + or :meth:`.BookmarkManager.get_all_bookmarks` gets called. + The function will be passed the name of the database (``str``) if + ``.get_bookmarks`` is called or ``None`` if ``.get_all_bookmarks`` + is called. The function must return a :class:`.Bookmarks` object. + The result of ``bookmark_supplier`` will then be concatenated with + the internal set of bookmarks and used to configure the session in + creation. + :param bookmarks_consumer: Function which will be called whenever the set of bookmarks handled by the bookmark manager gets updated with the new - internal bookmark set. + internal bookmark set. It will receive the name of the database + and the new set of bookmarks. :returns: A default implementation of :class:`BookmarkManager`. @@ -263,7 +268,7 @@ def bookmark_manager( return Neo4jBookmarkManager( initial_bookmarks=initial_bookmarks, bookmark_supplier=bookmark_supplier, - notify_bookmarks=notify_bookmarks + bookmarks_consumer=bookmarks_consumer ) @classmethod diff --git a/neo4j/api.py b/neo4j/api.py index c90f36bc4..3c0d61422 100644 --- a/neo4j/api.py +++ b/neo4j/api.py @@ -430,19 +430,8 @@ def get_bookmarks(self, database: str) -> t.Collection[str]: ... @abc.abstractmethod - def get_all_bookmarks( - self, must_included_databases: t.Iterable[str] - ) -> t.Collection[str]: - """Return all bookmarks. - - The prototypical implementation of this method iterates over all known - databases plus the ones provided in the `must_included_databases` - parameter and calls :meth:`get_bookmarks` for each of them. It then - returns the union of all the bookmarks. - - :param must_included_databases: - The databases which must be included in the result even if they - don't have been initialized yet. + def get_all_bookmarks(self) -> t.Collection[str]: + """Return all bookmarks for all known databases. :returns: The collected bookmarks. """ @@ -486,9 +475,7 @@ async def get_bookmarks(self, database: str) -> t.Collection[str]: get_bookmarks.__doc__ = BookmarkManager.get_bookmarks.__doc__ @abc.abstractmethod - async def get_all_bookmarks( - self, must_included_databases: t.Iterable[str] - ) -> t.Collection[str]: + async def get_all_bookmarks(self) -> t.Collection[str]: ... get_all_bookmarks.__doc__ = BookmarkManager.get_all_bookmarks.__doc__ From bb0efaea800af89c737c787a2a38cd28a9984cd8 Mon Sep 17 00:00:00 2001 From: Rouven Bauer Date: Fri, 12 Aug 2022 09:55:04 +0200 Subject: [PATCH 10/26] Make Bookmarks not iterable --- neo4j/_async/bookmark_manager.py | 17 +++++++++++++---- neo4j/_sync/bookmark_manager.py | 17 +++++++++++++---- neo4j/api.py | 11 ----------- 3 files changed, 26 insertions(+), 19 deletions(-) diff --git a/neo4j/_async/bookmark_manager.py b/neo4j/_async/bookmark_manager.py index 4ffab861c..6f5459b77 100644 --- a/neo4j/_async/bookmark_manager.py +++ b/neo4j/_async/bookmark_manager.py @@ -34,10 +34,19 @@ T_BmConsumer = t.Callable[[str, Bookmarks], t.Union[None, t.Awaitable[None]]] +def _bookmarks_to_set( + bookmarks: t.Union[Bookmarks, t.Iterable[str]] +) -> t.Set[str]: + if isinstance(bookmarks, Bookmarks): + return set(bookmarks.raw_values) + return set(map(str, bookmarks)) + + class AsyncNeo4jBookmarkManager(AsyncBookmarkManager): def __init__( self, - initial_bookmarks: t.Mapping[str, t.Iterable[str]] = None, + initial_bookmarks: t.Mapping[str, t.Union[Bookmarks, + t.Iterable[str]]] = None, bookmark_supplier: T_BmSupplier = None, bookmarks_consumer: T_BmConsumer = None ) -> None: @@ -47,7 +56,7 @@ def __init__( if initial_bookmarks is None: initial_bookmarks = {} self._bookmarks = defaultdict( - set, ((k, set(map(str, v))) + set, ((k, _bookmarks_to_set(v)) for k, v in initial_bookmarks.items()) ) self._lock = AsyncCooperativeLock() @@ -78,7 +87,7 @@ async def get_bookmarks(self, database: str) -> t.Set[str]: extra_bms = await AsyncUtil.callback( self._bookmark_supplier, database ) - bms.update(extra_bms) + bms.update(extra_bms.raw_values) return bms async def get_all_bookmarks(self) -> t.Set[str]: @@ -90,7 +99,7 @@ async def get_all_bookmarks(self) -> t.Set[str]: extra_bms = await AsyncUtil.callback( self._bookmark_supplier, None ) - bms.update(extra_bms) + bms.update(extra_bms.raw_values) return bms async def forget(self, databases: t.Iterable[str]) -> None: diff --git a/neo4j/_sync/bookmark_manager.py b/neo4j/_sync/bookmark_manager.py index 4a46614a0..c77056cb5 100644 --- a/neo4j/_sync/bookmark_manager.py +++ b/neo4j/_sync/bookmark_manager.py @@ -34,10 +34,19 @@ T_BmConsumer = t.Callable[[str, Bookmarks], t.Union[None, t.Union[None]]] +def _bookmarks_to_set( + bookmarks: t.Union[Bookmarks, t.Iterable[str]] +) -> t.Set[str]: + if isinstance(bookmarks, Bookmarks): + return set(bookmarks.raw_values) + return set(map(str, bookmarks)) + + class Neo4jBookmarkManager(BookmarkManager): def __init__( self, - initial_bookmarks: t.Mapping[str, t.Iterable[str]] = None, + initial_bookmarks: t.Mapping[str, t.Union[Bookmarks, + t.Iterable[str]]] = None, bookmark_supplier: T_BmSupplier = None, bookmarks_consumer: T_BmConsumer = None ) -> None: @@ -47,7 +56,7 @@ def __init__( if initial_bookmarks is None: initial_bookmarks = {} self._bookmarks = defaultdict( - set, ((k, set(map(str, v))) + set, ((k, _bookmarks_to_set(v)) for k, v in initial_bookmarks.items()) ) self._lock = CooperativeLock() @@ -78,7 +87,7 @@ def get_bookmarks(self, database: str) -> t.Set[str]: extra_bms = Util.callback( self._bookmark_supplier, database ) - bms.update(extra_bms) + bms.update(extra_bms.raw_values) return bms def get_all_bookmarks(self) -> t.Set[str]: @@ -90,7 +99,7 @@ def get_all_bookmarks(self) -> t.Set[str]: extra_bms = Util.callback( self._bookmark_supplier, None ) - bms.update(extra_bms) + bms.update(extra_bms.raw_values) return bms def forget(self, databases: t.Iterable[str]) -> None: diff --git a/neo4j/api.py b/neo4j/api.py index 3c0d61422..4148c884e 100644 --- a/neo4j/api.py +++ b/neo4j/api.py @@ -249,17 +249,6 @@ def __add__(self, other: Bookmarks) -> Bookmarks: return ret return NotImplemented - def __iter__(self) -> t.Iterator[str]: - """Iterate over the raw bookmark values. - - This is equivalent to:: - - bookmarks.raw_values.__iter__() - - :return: iterator over the raw bookmark values - """ - return iter(self._raw_values) - @property def raw_values(self) -> t.FrozenSet[str]: """The raw bookmark values. From ecd70db64e0fa40bc941b32f24fab7baf835ea2b Mon Sep 17 00:00:00 2001 From: Rouven Bauer Date: Fri, 12 Aug 2022 12:38:34 +0200 Subject: [PATCH 11/26] TestKit backend support for BMM extension functions --- testkitbackend/_async/backend.py | 2 + testkitbackend/_async/requests.py | 73 ++++++++++++++++++++++++++----- testkitbackend/_sync/backend.py | 2 + testkitbackend/_sync/requests.py | 73 ++++++++++++++++++++++++++----- 4 files changed, 126 insertions(+), 24 deletions(-) diff --git a/testkitbackend/_async/backend.py b/testkitbackend/_async/backend.py index c6aa85e6a..176b8f598 100644 --- a/testkitbackend/_async/backend.py +++ b/testkitbackend/_async/backend.py @@ -55,6 +55,8 @@ def __init__(self, rd, wr): self.drivers = {} self.custom_resolutions = {} self.dns_resolutions = {} + self.bookmark_consumptions = {} + self.bookmark_supplies = {} self.sessions = {} self.results = {} self.errors = {} diff --git a/testkitbackend/_async/requests.py b/testkitbackend/_async/requests.py index 9d8b22933..d3558ef53 100644 --- a/testkitbackend/_async/requests.py +++ b/testkitbackend/_async/requests.py @@ -22,6 +22,7 @@ from os import path import neo4j +import neo4j.api from neo4j._async_compat.util import AsyncUtil from .. import ( @@ -147,21 +148,19 @@ async def NewDriver(backend, data): data.mark_item_as_read_if_equals("livenessCheckTimeoutMs", None) bookmark_manager_config = data.get("bookmarkManager", {}) if bookmark_manager_config: - bookmark_manager_config.mark_item_as_read_if_equals( - "bookmarkSupplier", False - ) - bookmark_manager_config.mark_item_as_read_if_equals( - "notifyBookmarks", False - ) + bmm_kwargs = {} bookmark_manager_config.mark_item_as_read("initialBookmarks", recursive=True) - kwargs["bookmark_manager"] = neo4j.AsyncGraphDatabase.bookmark_manager( - initial_bookmarks=bookmark_manager_config.get("initialBookmarks"), - bookmark_supplier=None, - notify_bookmarks=None, - ) + bmm_kwargs["initial_bookmarks"] = \ + bookmark_manager_config.get("initialBookmarks") + if bookmark_manager_config.get("bookmarksSupplierRegistered"): + bmm_kwargs["bookmark_supplier"] = bookmark_supplier(backend) + if bookmark_manager_config.get("bookmarksConsumerRegistered"): + bmm_kwargs["bookmarks_consumer"] = bookmark_consumer(backend) + + kwargs["bookmark_manager"] = \ + neo4j.AsyncGraphDatabase.bookmark_manager(**bmm_kwargs) - data.mark_item_as_read("domainNameResolverRegistered") driver = neo4j.AsyncGraphDatabase.driver( data["uri"], auth=auth, user_agent=data["userAgent"], **kwargs ) @@ -258,6 +257,56 @@ async def DomainNameResolutionCompleted(backend, data): backend.dns_resolutions[data["requestId"]] = data["addresses"] +def bookmark_supplier(backend): + async def supplier(database): + key = backend.next_key() + await backend.send_response("BookmarksSupplierRequest", { + "id": key, + "database": database + }) + if not await backend.process_request(): + # connection was closed before end of next message + return [] + if key not in backend.bookmark_supplies: + raise RuntimeError( + "Backend did not receive expected " + "BookmarksSupplierCompleted message for id %s" % key + ) + return backend.bookmark_supplies.pop(key) + + return supplier + + +async def BookmarksSupplierCompleted(backend, data): + backend.bookmark_supplies[data["requestId"]] = \ + neo4j.Bookmarks.from_raw_values(data["bookmarks"]) + + +def bookmark_consumer(backend): + async def consumer(database, bookmarks): + key = backend.next_key() + await backend.send_response("BookmarksConsumerRequest", { + "id": key, + "database": database, + "bookmarks": list(bookmarks.raw_values) + }) + if not await backend.process_request(): + # connection was closed before end of next message + return [] + if key not in backend.bookmark_consumptions: + raise RuntimeError( + "Backend did not receive expected " + "BookmarksConsumerCompleted message for id %s" % key + ) + del backend.bookmark_consumptions[key] + + return consumer + + +async def BookmarksConsumerCompleted(backend, data): + backend.bookmark_consumptions[data["requestId"]] = True + + async def DriverClose(backend, data): key = data["driverId"] driver = backend.drivers[key] diff --git a/testkitbackend/_sync/backend.py b/testkitbackend/_sync/backend.py index a07a89d50..5d9b5b8f3 100644 --- a/testkitbackend/_sync/backend.py +++ b/testkitbackend/_sync/backend.py @@ -55,6 +55,8 @@ def __init__(self, rd, wr): self.drivers = {} self.custom_resolutions = {} self.dns_resolutions = {} + self.bookmark_consumptions = {} + self.bookmark_supplies = {} self.sessions = {} self.results = {} self.errors = {} diff --git a/testkitbackend/_sync/requests.py b/testkitbackend/_sync/requests.py index 549d1e299..0fed2f19d 100644 --- a/testkitbackend/_sync/requests.py +++ b/testkitbackend/_sync/requests.py @@ -22,6 +22,7 @@ from os import path import neo4j +import neo4j.api from neo4j._async_compat.util import Util from .. import ( @@ -147,21 +148,19 @@ def NewDriver(backend, data): data.mark_item_as_read_if_equals("livenessCheckTimeoutMs", None) bookmark_manager_config = data.get("bookmarkManager", {}) if bookmark_manager_config: - bookmark_manager_config.mark_item_as_read_if_equals( - "bookmarkSupplier", False - ) - bookmark_manager_config.mark_item_as_read_if_equals( - "notifyBookmarks", False - ) + bmm_kwargs = {} bookmark_manager_config.mark_item_as_read("initialBookmarks", recursive=True) - kwargs["bookmark_manager"] = neo4j.GraphDatabase.bookmark_manager( - initial_bookmarks=bookmark_manager_config.get("initialBookmarks"), - bookmark_supplier=None, - notify_bookmarks=None, - ) + bmm_kwargs["initial_bookmarks"] = \ + bookmark_manager_config.get("initialBookmarks") + if bookmark_manager_config.get("bookmarksSupplierRegistered"): + bmm_kwargs["bookmark_supplier"] = bookmark_supplier(backend) + if bookmark_manager_config.get("bookmarksConsumerRegistered"): + bmm_kwargs["bookmarks_consumer"] = bookmark_consumer(backend) + + kwargs["bookmark_manager"] = \ + neo4j.GraphDatabase.bookmark_manager(**bmm_kwargs) - data.mark_item_as_read("domainNameResolverRegistered") driver = neo4j.GraphDatabase.driver( data["uri"], auth=auth, user_agent=data["userAgent"], **kwargs ) @@ -258,6 +257,56 @@ def DomainNameResolutionCompleted(backend, data): backend.dns_resolutions[data["requestId"]] = data["addresses"] +def bookmark_supplier(backend): + def supplier(database): + key = backend.next_key() + backend.send_response("BookmarksSupplierRequest", { + "id": key, + "database": database + }) + if not backend.process_request(): + # connection was closed before end of next message + return [] + if key not in backend.bookmark_supplies: + raise RuntimeError( + "Backend did not receive expected " + "BookmarksSupplierCompleted message for id %s" % key + ) + return backend.bookmark_supplies.pop(key) + + return supplier + + +def BookmarksSupplierCompleted(backend, data): + backend.bookmark_supplies[data["requestId"]] = \ + neo4j.Bookmarks.from_raw_values(data["bookmarks"]) + + +def bookmark_consumer(backend): + def consumer(database, bookmarks): + key = backend.next_key() + backend.send_response("BookmarksConsumerRequest", { + "id": key, + "database": database, + "bookmarks": list(bookmarks.raw_values) + }) + if not backend.process_request(): + # connection was closed before end of next message + return [] + if key not in backend.bookmark_consumptions: + raise RuntimeError( + "Backend did not receive expected " + "BookmarksConsumerCompleted message for id %s" % key + ) + del backend.bookmark_consumptions[key] + + return consumer + + +def BookmarksConsumerCompleted(backend, data): + backend.bookmark_consumptions[data["requestId"]] = True + + def DriverClose(backend, data): key = data["driverId"] driver = backend.drivers[key] From 90d6abcfc1f7cd040e18d24123a11d5461476507 Mon Sep 17 00:00:00 2001 From: Rouven Bauer Date: Fri, 12 Aug 2022 12:42:44 +0200 Subject: [PATCH 12/26] Fix corner-case usages of the BMM within Session --- neo4j/_async/io/_pool.py | 6 +++--- neo4j/_async/work/session.py | 5 +---- neo4j/_async/work/workspace.py | 26 +++++++++++++++++--------- neo4j/_sync/io/_pool.py | 6 +++--- neo4j/_sync/work/session.py | 5 +---- neo4j/_sync/work/workspace.py | 26 +++++++++++++++++--------- 6 files changed, 42 insertions(+), 32 deletions(-) diff --git a/neo4j/_async/io/_pool.py b/neo4j/_async/io/_pool.py index 82ff5ed60..b530b3a2a 100644 --- a/neo4j/_async/io/_pool.py +++ b/neo4j/_async/io/_pool.py @@ -515,9 +515,9 @@ async def fetch_routing_info( cx = await self._acquire(address, deadline, None) try: routing_table = await cx.route( - database or self.workspace_config.database, - imp_user or self.workspace_config.impersonated_user, - bookmarks + database=database or self.workspace_config.database, + imp_user=imp_user or self.workspace_config.impersonated_user, + bookmarks=bookmarks ) finally: await self.release(cx) diff --git a/neo4j/_async/work/session.py b/neo4j/_async/work/session.py index 67e526f38..8960cc828 100644 --- a/neo4j/_async/work/session.py +++ b/neo4j/_async/work/session.py @@ -36,10 +36,7 @@ from ..._async_compat import async_sleep from ..._async_compat.util import AsyncUtil from ..._conf import SessionConfig -from ..._meta import ( - deprecated, - deprecation_warn, -) +from ..._meta import deprecated from ...api import ( Bookmarks, READ_ACCESS, diff --git a/neo4j/_async/work/workspace.py b/neo4j/_async/work/workspace.py index 69bc9ca73..3c43c0f37 100644 --- a/neo4j/_async/work/workspace.py +++ b/neo4j/_async/work/workspace.py @@ -99,25 +99,28 @@ def _initialize_bookmarks(self, bookmarks): "iterable of raw bookmarks (deprecated).") self._initial_bookmarks = self._bookmarks = prepared_bookmarks - async def _get_system_bookmarks(self): + async def _get_bookmarks(self, database): if self._bookmark_manager is not None: + # For 4.3- support: the server will not send the resolved home + # database back. To avoid confusion between `None` as in "all + # database" and `None` as in "home database" we re-write the + # home database to `""`, which otherwise is an invalid database + # name. + if database is None: + database = "" self._bookmarks = tuple({ *await AsyncUtil.callback( - self._bookmark_manager.get_bookmarks, "system" + self._bookmark_manager.get_bookmarks, database ), *self._initial_bookmarks }) return self._bookmarks async def _get_all_bookmarks(self): - must_included_databases = ["system"] - if self._config.database: - must_included_databases.append(self._config.database) if self._bookmark_manager is not None: self._bookmarks = tuple({ *await AsyncUtil.callback( self._bookmark_manager.get_all_bookmarks, - must_included_databases ), *self._initial_bookmarks }) @@ -131,6 +134,12 @@ async def _update_bookmarks(self, database, new_bookmarks): self._bookmarks = new_bookmarks if self._bookmark_manager is None: return + # For 4.3- support: the server will not send the resolved home + # database back. To avoid confusion between `None` as in "all + # database" and `None` as in "home database" we re-write the home + # database to `""`, which otherwise is an invalid database name. + if database is None: + database = "" await AsyncUtil.callback( self._bookmark_manager.update_bookmarks, database, previous_bookmarks, new_bookmarks @@ -162,11 +171,10 @@ async def _connect(self, access_mode, **acquire_kwargs): # to try to fetch the home database. If provided by the server, # we shall use this database explicitly for all subsequent # actions within this session. - bookmarks = await self._get_system_bookmarks() await self._pool.update_routing_table( database=self._config.database, imp_user=self._config.impersonated_user, - bookmarks=bookmarks, + bookmarks=await self._get_bookmarks("system"), acquisition_timeout=acquisition_timeout, database_callback=self._set_cached_database ) @@ -174,7 +182,7 @@ async def _connect(self, access_mode, **acquire_kwargs): "access_mode": access_mode, "timeout": acquisition_timeout, "database": self._config.database, - "bookmarks": self._bookmarks, + "bookmarks": await self._get_bookmarks("system"), "liveness_check_timeout": None, } acquire_kwargs_.update(acquire_kwargs) diff --git a/neo4j/_sync/io/_pool.py b/neo4j/_sync/io/_pool.py index d3ea665f4..947555a01 100644 --- a/neo4j/_sync/io/_pool.py +++ b/neo4j/_sync/io/_pool.py @@ -515,9 +515,9 @@ def fetch_routing_info( cx = self._acquire(address, deadline, None) try: routing_table = cx.route( - database or self.workspace_config.database, - imp_user or self.workspace_config.impersonated_user, - bookmarks + database=database or self.workspace_config.database, + imp_user=imp_user or self.workspace_config.impersonated_user, + bookmarks=bookmarks ) finally: self.release(cx) diff --git a/neo4j/_sync/work/session.py b/neo4j/_sync/work/session.py index 927db4e93..000454e3c 100644 --- a/neo4j/_sync/work/session.py +++ b/neo4j/_sync/work/session.py @@ -36,10 +36,7 @@ from ..._async_compat import sleep from ..._async_compat.util import Util from ..._conf import SessionConfig -from ..._meta import ( - deprecated, - deprecation_warn, -) +from ..._meta import deprecated from ...api import ( Bookmarks, READ_ACCESS, diff --git a/neo4j/_sync/work/workspace.py b/neo4j/_sync/work/workspace.py index 950490776..054d4b41c 100644 --- a/neo4j/_sync/work/workspace.py +++ b/neo4j/_sync/work/workspace.py @@ -99,25 +99,28 @@ def _initialize_bookmarks(self, bookmarks): "iterable of raw bookmarks (deprecated).") self._initial_bookmarks = self._bookmarks = prepared_bookmarks - def _get_system_bookmarks(self): + def _get_bookmarks(self, database): if self._bookmark_manager is not None: + # For 4.3- support: the server will not send the resolved home + # database back. To avoid confusion between `None` as in "all + # database" and `None` as in "home database" we re-write the + # home database to `""`, which otherwise is an invalid database + # name. + if database is None: + database = "" self._bookmarks = tuple({ *Util.callback( - self._bookmark_manager.get_bookmarks, "system" + self._bookmark_manager.get_bookmarks, database ), *self._initial_bookmarks }) return self._bookmarks def _get_all_bookmarks(self): - must_included_databases = ["system"] - if self._config.database: - must_included_databases.append(self._config.database) if self._bookmark_manager is not None: self._bookmarks = tuple({ *Util.callback( self._bookmark_manager.get_all_bookmarks, - must_included_databases ), *self._initial_bookmarks }) @@ -131,6 +134,12 @@ def _update_bookmarks(self, database, new_bookmarks): self._bookmarks = new_bookmarks if self._bookmark_manager is None: return + # For 4.3- support: the server will not send the resolved home + # database back. To avoid confusion between `None` as in "all + # database" and `None` as in "home database" we re-write the home + # database to `""`, which otherwise is an invalid database name. + if database is None: + database = "" Util.callback( self._bookmark_manager.update_bookmarks, database, previous_bookmarks, new_bookmarks @@ -162,11 +171,10 @@ def _connect(self, access_mode, **acquire_kwargs): # to try to fetch the home database. If provided by the server, # we shall use this database explicitly for all subsequent # actions within this session. - bookmarks = self._get_system_bookmarks() self._pool.update_routing_table( database=self._config.database, imp_user=self._config.impersonated_user, - bookmarks=bookmarks, + bookmarks=self._get_bookmarks("system"), acquisition_timeout=acquisition_timeout, database_callback=self._set_cached_database ) @@ -174,7 +182,7 @@ def _connect(self, access_mode, **acquire_kwargs): "access_mode": access_mode, "timeout": acquisition_timeout, "database": self._config.database, - "bookmarks": self._bookmarks, + "bookmarks": self._get_bookmarks("system"), "liveness_check_timeout": None, } acquire_kwargs_.update(acquire_kwargs) From bb07e19a45b06e5ac5dd0079af290e616b29d414 Mon Sep 17 00:00:00 2001 From: Rouven Bauer Date: Wed, 17 Aug 2022 15:11:36 +0200 Subject: [PATCH 13/26] Unit tests for BMM --- bin/make-unasync | 5 +- docs/source/api.rst | 3 +- docs/source/async_api.rst | 3 +- neo4j/_async/bookmark_manager.py | 16 +- neo4j/_async/driver.py | 6 +- neo4j/_async_compat/concurrency.py | 14 +- neo4j/_async_compat/util.py | 12 +- neo4j/_sync/bookmark_manager.py | 16 +- neo4j/_sync/driver.py | 6 +- neo4j/_sync/work/session.py | 2 +- neo4j/api.py | 19 +- tests/_async_compat/__init__.py | 4 + tests/_async_compat/mark_decorator.py | 11 + tests/unit/async_/conftest.py | 19 ++ tests/unit/async_/fixtures/__init__.py | 20 ++ .../fake_connection.py} | 30 +- tests/unit/async_/fixtures/fake_pool.py | 45 +++ tests/unit/async_/io/test_class_bolt4x3.py | 2 +- tests/unit/async_/io/test_neo4j_pool.py | 1 - tests/unit/async_/test_addressing.py | 28 +- tests/unit/async_/test_bookmark_manager.py | 295 ++++++++++++++++++ tests/unit/async_/test_driver.py | 222 +++++++++++-- tests/unit/async_/work/__init__.py | 6 - tests/unit/async_/work/conftest.py | 6 - tests/unit/async_/work/test_session.py | 222 ++++++++++--- tests/unit/common/test_conf.py | 1 + tests/unit/sync/conftest.py | 19 ++ tests/unit/sync/fixtures/__init__.py | 20 ++ tests/unit/sync/fixtures/fake_connection.py | 215 +++++++++++++ tests/unit/sync/fixtures/fake_pool.py | 45 +++ tests/unit/sync/io/test_neo4j_pool.py | 1 - tests/unit/sync/test_addressing.py | 28 +- tests/unit/sync/test_bookmark_manager.py | 295 ++++++++++++++++++ tests/unit/sync/test_driver.py | 221 +++++++++++-- tests/unit/sync/work/__init__.py | 6 - tests/unit/sync/work/_fake_connection.py | 22 +- tests/unit/sync/work/test_session.py | 222 ++++++++++--- 37 files changed, 1835 insertions(+), 273 deletions(-) create mode 100644 tests/unit/async_/conftest.py create mode 100644 tests/unit/async_/fixtures/__init__.py rename tests/unit/async_/{work/_fake_connection.py => fixtures/fake_connection.py} (92%) create mode 100644 tests/unit/async_/fixtures/fake_pool.py create mode 100644 tests/unit/async_/test_bookmark_manager.py delete mode 100644 tests/unit/async_/work/conftest.py create mode 100644 tests/unit/sync/conftest.py create mode 100644 tests/unit/sync/fixtures/__init__.py create mode 100644 tests/unit/sync/fixtures/fake_connection.py create mode 100644 tests/unit/sync/fixtures/fake_pool.py create mode 100644 tests/unit/sync/test_bookmark_manager.py diff --git a/bin/make-unasync b/bin/make-unasync index f1b99465f..675a6bc4c 100755 --- a/bin/make-unasync +++ b/bin/make-unasync @@ -154,12 +154,12 @@ class CustomRule(unasync.Rule): start += 1 end += 1 else: - out += self._unasync_prefix(name[start:(end - 1)]) + out += self._unasync_name(name[start:(end - 1)]) start = end - 1 sub_name = name[start:] if sub_name.isidentifier(): - out += self._unasync_prefix(name[start:]) + out += self._unasync_name(name[start:]) else: out += sub_name @@ -221,6 +221,7 @@ def apply_unasync(files): "mark_async_test": "mark_sync_test", "assert_awaited_once": "assert_called_once", "assert_awaited_once_with": "assert_called_once_with", + "await_count": "call_count", } additional_testkit_backend_replacements = {} rules = [ diff --git a/docs/source/api.rst b/docs/source/api.rst index 4c9b832db..caa9a03cf 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -842,7 +842,7 @@ Returning a live result object would prevent the driver from correctly managing This function will receive a :class:`neo4j.ManagedTransaction` object as its first parameter. -.. autoclass:: neo4j.ManagedTransaction +.. autoclass:: neo4j.ManagedTransaction() .. automethod:: run @@ -1249,7 +1249,6 @@ BookmarkManager *************** .. autoclass:: neo4j.api.BookmarkManager - :show-inheritance: :members: diff --git a/docs/source/async_api.rst b/docs/source/async_api.rst index eefe1e6fa..28c367a8e 100644 --- a/docs/source/async_api.rst +++ b/docs/source/async_api.rst @@ -525,7 +525,7 @@ Returning a live result object would prevent the driver from correctly managing This function will receive a :class:`neo4j.AsyncManagedTransaction` object as its first parameter. -.. autoclass:: neo4j.AsyncManagedTransaction +.. autoclass:: neo4j.AsyncManagedTransaction() .. automethod:: run @@ -596,7 +596,6 @@ AsyncBookmarkManager ******************** .. autoclass:: neo4j.api.AsyncBookmarkManager - :show-inheritance: :members: diff --git a/neo4j/_async/bookmark_manager.py b/neo4j/_async/bookmark_manager.py index 6f5459b77..d24d381be 100644 --- a/neo4j/_async/bookmark_manager.py +++ b/neo4j/_async/bookmark_manager.py @@ -48,11 +48,11 @@ def __init__( initial_bookmarks: t.Mapping[str, t.Union[Bookmarks, t.Iterable[str]]] = None, bookmark_supplier: T_BmSupplier = None, - bookmarks_consumer: T_BmConsumer = None + bookmark_consumer: T_BmConsumer = None ) -> None: super().__init__() self._bookmark_supplier = bookmark_supplier - self._bookmarks_consumer = bookmarks_consumer + self._bookmark_consumer = bookmark_consumer if initial_bookmarks is None: initial_bookmarks = {} self._bookmarks = defaultdict( @@ -73,11 +73,11 @@ async def update_bookmarks( curr_bms = self._bookmarks[database] curr_bms.difference_update(prev_bms) curr_bms.update(new_bms) - if self._bookmarks_consumer: + if self._bookmark_consumer: curr_bms_snapshot = Bookmarks.from_raw_values(curr_bms) - if self._bookmarks_consumer: + if self._bookmark_consumer: await AsyncUtil.callback( - self._bookmarks_consumer, database, curr_bms_snapshot + self._bookmark_consumer, database, curr_bms_snapshot ) async def get_bookmarks(self, database: str) -> t.Set[str]: @@ -103,6 +103,6 @@ async def get_all_bookmarks(self) -> t.Set[str]: return bms async def forget(self, databases: t.Iterable[str]) -> None: - for database in databases: - with self._lock: - del self._bookmarks[database] + with self._lock: + for database in databases: + self._bookmarks.pop(database, None) diff --git a/neo4j/_async/driver.py b/neo4j/_async/driver.py index 887da1423..df846b87b 100644 --- a/neo4j/_async/driver.py +++ b/neo4j/_async/driver.py @@ -221,7 +221,7 @@ def bookmark_manager( initial_bookmarks: t.Mapping[str, t.Union[Bookmarks, t.Iterable[str]]] = None, bookmark_supplier: _T_BmSupplier = None, - bookmarks_consumer: _T_BmConsumer = None + bookmark_consumer: _T_BmConsumer = None ) -> AsyncBookmarkManager: """Create a default :class:`.AsyncBookmarkManager`. @@ -254,7 +254,7 @@ def bookmark_manager( The result of ``bookmark_supplier`` will then be concatenated with the internal set of bookmarks and used to configure the session in creation. - :param bookmarks_consumer: + :param bookmark_consumer: Function which will be called whenever the set of bookmarks handled by the bookmark manager gets updated with the new internal bookmark set. It will receive the name of the database @@ -267,7 +267,7 @@ def bookmark_manager( return AsyncNeo4jBookmarkManager( initial_bookmarks=initial_bookmarks, bookmark_supplier=bookmark_supplier, - bookmarks_consumer=bookmarks_consumer + bookmark_consumer=bookmark_consumer ) @classmethod diff --git a/neo4j/_async_compat/concurrency.py b/neo4j/_async_compat/concurrency.py index 96611f72a..6c6145438 100644 --- a/neo4j/_async_compat/concurrency.py +++ b/neo4j/_async_compat/concurrency.py @@ -24,6 +24,10 @@ import threading import typing as t + +if t.TYPE_CHECKING: + import typing_extensions as te + from neo4j._async_compat.shims import wait_for @@ -432,8 +436,8 @@ def notify_all(self): self.notify(len(self._waiters)) -Condition: t.TypeAlias = threading.Condition -CooperativeLock: t.TypeAlias = threading.Lock -Lock: t.TypeAlias = threading.Lock -CooperativeRLock: t.TypeAlias = threading.RLock -RLock: t.TypeAlias = threading.RLock +Condition: te.TypeAlias = threading.Condition +CooperativeLock: te.TypeAlias = threading.Lock +Lock: te.TypeAlias = threading.Lock +CooperativeRLock: te.TypeAlias = threading.RLock +RLock: te.TypeAlias = threading.RLock diff --git a/neo4j/_async_compat/util.py b/neo4j/_async_compat/util.py index c7ed041ed..ac6a7ffaa 100644 --- a/neo4j/_async_compat/util.py +++ b/neo4j/_async_compat/util.py @@ -78,7 +78,7 @@ async def callback(cb, *args, **kwargs): return await res return res - experimental_async = experimental + experimental_async: t.ClassVar = experimental @staticmethod def shielded(coro_function): @@ -90,13 +90,13 @@ async def shielded_function(*args, **kwargs): return shielded_function - is_async_code = True + is_async_code: t.ClassVar = True class Util: - iter = iter - next = next - list = list + iter: t.ClassVar = iter + next: t.ClassVar = next + list: t.ClassVar = list @staticmethod @t.overload @@ -124,4 +124,4 @@ def f_(f): def shielded(coro_function): return coro_function - is_async_code = False + is_async_code: t.ClassVar = False diff --git a/neo4j/_sync/bookmark_manager.py b/neo4j/_sync/bookmark_manager.py index c77056cb5..9ae230fa6 100644 --- a/neo4j/_sync/bookmark_manager.py +++ b/neo4j/_sync/bookmark_manager.py @@ -48,11 +48,11 @@ def __init__( initial_bookmarks: t.Mapping[str, t.Union[Bookmarks, t.Iterable[str]]] = None, bookmark_supplier: T_BmSupplier = None, - bookmarks_consumer: T_BmConsumer = None + bookmark_consumer: T_BmConsumer = None ) -> None: super().__init__() self._bookmark_supplier = bookmark_supplier - self._bookmarks_consumer = bookmarks_consumer + self._bookmark_consumer = bookmark_consumer if initial_bookmarks is None: initial_bookmarks = {} self._bookmarks = defaultdict( @@ -73,11 +73,11 @@ def update_bookmarks( curr_bms = self._bookmarks[database] curr_bms.difference_update(prev_bms) curr_bms.update(new_bms) - if self._bookmarks_consumer: + if self._bookmark_consumer: curr_bms_snapshot = Bookmarks.from_raw_values(curr_bms) - if self._bookmarks_consumer: + if self._bookmark_consumer: Util.callback( - self._bookmarks_consumer, database, curr_bms_snapshot + self._bookmark_consumer, database, curr_bms_snapshot ) def get_bookmarks(self, database: str) -> t.Set[str]: @@ -103,6 +103,6 @@ def get_all_bookmarks(self) -> t.Set[str]: return bms def forget(self, databases: t.Iterable[str]) -> None: - for database in databases: - with self._lock: - del self._bookmarks[database] + with self._lock: + for database in databases: + self._bookmarks.pop(database, None) diff --git a/neo4j/_sync/driver.py b/neo4j/_sync/driver.py index d75861e7e..442e5a7f6 100644 --- a/neo4j/_sync/driver.py +++ b/neo4j/_sync/driver.py @@ -220,7 +220,7 @@ def bookmark_manager( initial_bookmarks: t.Mapping[str, t.Union[Bookmarks, t.Iterable[str]]] = None, bookmark_supplier: _T_BmSupplier = None, - bookmarks_consumer: _T_BmConsumer = None + bookmark_consumer: _T_BmConsumer = None ) -> BookmarkManager: """Create a default :class:`.BookmarkManager`. @@ -253,7 +253,7 @@ def bookmark_manager( The result of ``bookmark_supplier`` will then be concatenated with the internal set of bookmarks and used to configure the session in creation. - :param bookmarks_consumer: + :param bookmark_consumer: Function which will be called whenever the set of bookmarks handled by the bookmark manager gets updated with the new internal bookmark set. It will receive the name of the database @@ -266,7 +266,7 @@ def bookmark_manager( return Neo4jBookmarkManager( initial_bookmarks=initial_bookmarks, bookmark_supplier=bookmark_supplier, - bookmarks_consumer=bookmarks_consumer + bookmark_consumer=bookmark_consumer ) @classmethod diff --git a/neo4j/_sync/work/session.py b/neo4j/_sync/work/session.py index 000454e3c..946f6c9de 100644 --- a/neo4j/_sync/work/session.py +++ b/neo4j/_sync/work/session.py @@ -109,7 +109,7 @@ def __enter__(self) -> Session: def __exit__(self, exception_type, exception_value, traceback): if exception_type: if issubclass(exception_type, asyncio.CancelledError): - self._handle_cancellation(message="__aexit__") + self._handle_cancellation(message="__exit__") self._closed = True return self._state_failed = True diff --git a/neo4j/api.py b/neo4j/api.py index 4148c884e..f9a1faece 100644 --- a/neo4j/api.py +++ b/neo4j/api.py @@ -36,6 +36,12 @@ from .exceptions import ConfigurationError +if t.TYPE_CHECKING: + from typing_extensions import Protocol as _Protocol +else: + _Protocol = object + + READ_ACCESS: te.Final[str] = "READ" WRITE_ACCESS: te.Final[str] = "WRITE" @@ -366,7 +372,7 @@ def from_bytes(cls, b: bytes) -> Version: return Version(b[-1], b[-2]) -class BookmarkManager(abc.ABC): +class BookmarkManager(_Protocol, metaclass=abc.ABCMeta): """Class to manage bookmarks throughout the driver's lifetime. Neo4j clusters are eventually consistent, meaning that there is no @@ -383,12 +389,17 @@ class BookmarkManager(abc.ABC): consistent. Configure the driver to use a specific bookmark manager with :ref:`bookmark-manager-ref`. - The driver comes with a default implementation of the bookmark manager - accessible through :attr:`.GraphDatabase.bookmark_manager()`. + This class is just an abstract base class that defines the required + interface. Create a child class to implement a specific bookmark manager + or make user of the default implementation provided by the driver through + :meth:`.GraphDatabase.bookmark_manager()`. .. note:: All methods must be concurrency safe. + Generally, all methods need to be able to cope with getting passed a + ``database`` parameter that is (until then) unknown to the manager. + .. versionadded:: 5.0 """ @@ -439,7 +450,7 @@ def forget(self, databases: t.Iterable[str]) -> None: ... -class AsyncBookmarkManager(abc.ABC): +class AsyncBookmarkManager(_Protocol, metaclass=abc.ABCMeta): """Same as :class:`.BookmarkManager` but with async methods. The driver comes with a default implementation of the async bookmark diff --git a/tests/_async_compat/__init__.py b/tests/_async_compat/__init__.py index b9aec02c4..12a189840 100644 --- a/tests/_async_compat/__init__.py +++ b/tests/_async_compat/__init__.py @@ -17,12 +17,16 @@ from .mark_decorator import ( + AsyncTestDecorators, mark_async_test, mark_sync_test, + TestDecorators, ) __all__ = [ + "AsyncTestDecorators", "mark_async_test", "mark_sync_test", + "TestDecorators", ] diff --git a/tests/_async_compat/mark_decorator.py b/tests/_async_compat/mark_decorator.py index cfb42f453..b89754cb2 100644 --- a/tests/_async_compat/mark_decorator.py +++ b/tests/_async_compat/mark_decorator.py @@ -24,3 +24,14 @@ def mark_sync_test(f): return f + + +class AsyncTestDecorators: + mark_async_only_test = mark_async_test + + +class TestDecorators: + @staticmethod + def mark_async_only_test(f): + skip_decorator = pytest.mark.skip("Async only test") + return skip_decorator(f) diff --git a/tests/unit/async_/conftest.py b/tests/unit/async_/conftest.py new file mode 100644 index 000000000..9d171987d --- /dev/null +++ b/tests/unit/async_/conftest.py @@ -0,0 +1,19 @@ +# Copyright (c) "Neo4j" +# Neo4j Sweden AB [https://neo4j.com] +# +# This file is part of Neo4j. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .fixtures import * # necessary for pytest to discover the fixtures diff --git a/tests/unit/async_/fixtures/__init__.py b/tests/unit/async_/fixtures/__init__.py new file mode 100644 index 000000000..c3e907d44 --- /dev/null +++ b/tests/unit/async_/fixtures/__init__.py @@ -0,0 +1,20 @@ +# Copyright (c) "Neo4j" +# Neo4j Sweden AB [https://neo4j.com] +# +# This file is part of Neo4j. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .fake_connection import * +from .fake_pool import * diff --git a/tests/unit/async_/work/_fake_connection.py b/tests/unit/async_/fixtures/fake_connection.py similarity index 92% rename from tests/unit/async_/work/_fake_connection.py rename to tests/unit/async_/fixtures/fake_connection.py index 0966b1556..f0d0070c1 100644 --- a/tests/unit/async_/work/_fake_connection.py +++ b/tests/unit/async_/fixtures/fake_connection.py @@ -25,6 +25,14 @@ from neo4j._deadline import Deadline +__all__ = [ + "async_fake_connection_generator", + "async_fake_connection", + "async_scripted_connection_generator", + "async_scripted_connection", +] + + @pytest.fixture def async_fake_connection_generator(session_mocker): mock = session_mocker.mock_module @@ -158,15 +166,15 @@ def __getattr__(self, name): parent = super() def build_message_handler(name): - try: - expected_message, scripted_callbacks = \ - self._script[self._script_pos] - except IndexError: - pytest.fail("End of scripted connection reached.") - assert name == expected_message - self._script_pos += 1 - def func(*args, **kwargs): + try: + expected_message, scripted_callbacks = \ + self._script[self._script_pos] + except IndexError: + pytest.fail("End of scripted connection reached.") + assert name == expected_message + self._script_pos += 1 + async def callback(): for cb_name, default_cb_args in ( ("on_ignored", ({},)), @@ -176,8 +184,10 @@ async def callback(): ("on_summary", ()), ): cb = kwargs.get(cb_name, None) - if (not callable(cb) - or cb_name not in scripted_callbacks): + if ( + not callable(cb) + or cb_name not in scripted_callbacks + ): continue cb_args = scripted_callbacks[cb_name] if cb_args is None: diff --git a/tests/unit/async_/fixtures/fake_pool.py b/tests/unit/async_/fixtures/fake_pool.py new file mode 100644 index 000000000..ba4636f7e --- /dev/null +++ b/tests/unit/async_/fixtures/fake_pool.py @@ -0,0 +1,45 @@ +# Copyright (c) "Neo4j" +# Neo4j Sweden AB [https://neo4j.com] +# +# This file is part of Neo4j. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytest + +from neo4j._async.io._pool import AsyncIOPool + + +__all__ = [ + "fake_pool", +] + + +@pytest.fixture +def fake_pool(async_fake_connection_generator, mocker): + pool = mocker.AsyncMock(spec=AsyncIOPool) + assert not hasattr(pool, "acquired_connection_mocks") + pool.buffered_connection_mocks = [] + pool.acquired_connection_mocks = [] + + def acquire_side_effect(*_, **__): + if pool.buffered_connection_mocks: + connection = pool.buffered_connection_mocks.pop() + else: + connection = async_fake_connection_generator() + pool.acquired_connection_mocks.append(connection) + return connection + + pool.acquire.side_effect = acquire_side_effect + return pool diff --git a/tests/unit/async_/io/test_class_bolt4x3.py b/tests/unit/async_/io/test_class_bolt4x3.py index 7538e127b..eb74241ca 100644 --- a/tests/unit/async_/io/test_class_bolt4x3.py +++ b/tests/unit/async_/io/test_class_bolt4x3.py @@ -230,7 +230,7 @@ async def test_hint_recv_timeout_seconds( sockets = fake_socket_pair(address, packer_cls=AsyncBolt4x3.PACKER_CLS, unpacker_cls=AsyncBolt4x3.UNPACKER_CLS) - sockets.client.settimeout = mocker.AsyncMock() + sockets.client.settimeout = mocker.Mock() await sockets.server.send_message( b"\x70", {"server": "Neo4j/4.3.0", "hints": hints} ) diff --git a/tests/unit/async_/io/test_neo4j_pool.py b/tests/unit/async_/io/test_neo4j_pool.py index 07b136551..b9bbc4e42 100644 --- a/tests/unit/async_/io/test_neo4j_pool.py +++ b/tests/unit/async_/io/test_neo4j_pool.py @@ -38,7 +38,6 @@ ) from ...._async_compat import mark_async_test -from ..work import async_fake_connection_generator # needed as fixture ROUTER_ADDRESS = ResolvedAddress(("1.2.3.1", 9001), host_name="host") diff --git a/tests/unit/async_/test_addressing.py b/tests/unit/async_/test_addressing.py index 0c3fcfc54..75a036f9c 100644 --- a/tests/unit/async_/test_addressing.py +++ b/tests/unit/async_/test_addressing.py @@ -34,7 +34,7 @@ @mark_async_test -async def test_address_resolve(): +async def test_address_resolve() -> None: address = Address(("127.0.0.1", 7687)) resolved = AsyncNetworkUtil.resolve_address(address) resolved = await AsyncUtil.list(resolved) @@ -45,7 +45,7 @@ async def test_address_resolve(): @mark_async_test -async def test_address_resolve_with_custom_resolver_none(): +async def test_address_resolve_with_custom_resolver_none() -> None: address = Address(("127.0.0.1", 7687)) resolved = AsyncNetworkUtil.resolve_address(address, resolver=None) resolved = await AsyncUtil.list(resolved) @@ -64,7 +64,9 @@ async def test_address_resolve_with_custom_resolver_none(): ) @mark_async_test -async def test_address_resolve_with_unresolvable_address(test_input, expected): +async def test_address_resolve_with_unresolvable_address( + test_input, expected +) -> None: with pytest.raises(expected): await AsyncUtil.list( AsyncNetworkUtil.resolve_address(test_input, resolver=None) @@ -73,7 +75,7 @@ async def test_address_resolve_with_unresolvable_address(test_input, expected): @mark_async_test @pytest.mark.parametrize("resolver_type", ("sync", "async")) -async def test_address_resolve_with_custom_resolver(resolver_type): +async def test_address_resolve_with_custom_resolver(resolver_type) -> None: def custom_resolver_sync(_): return [("127.0.0.1", 7687), ("localhost", 1234)] @@ -98,9 +100,11 @@ async def custom_resolver_async(_): @mark_async_test -async def test_address_unresolve(): +async def test_address_unresolve() -> None: + def custom_resolver(_): + return custom_resolved + custom_resolved = [("127.0.0.1", 7687), ("localhost", 4321)] - custom_resolver = lambda _: custom_resolved address = Address(("foobar", 1234)) unresolved = address.unresolved @@ -109,9 +113,9 @@ async def test_address_unresolve(): resolved = AsyncNetworkUtil.resolve_address( address, family=AF_INET, resolver=custom_resolver ) - resolved = await AsyncUtil.list(resolved) - custom_resolved = sorted(Address(a) for a in custom_resolved) - unresolved = sorted(a.unresolved for a in resolved) - assert custom_resolved == unresolved - assert (list(map(lambda a: a.__class__, custom_resolved)) - == list(map(lambda a: a.__class__, unresolved))) + resolved_list = await AsyncUtil.list(resolved) + custom_resolved_addresses = sorted(Address(a) for a in custom_resolved) + unresolved_list = sorted(a.unresolved for a in resolved_list) + assert custom_resolved_addresses == unresolved_list + assert (list(map(lambda a: a.__class__, custom_resolved_addresses)) + == list(map(lambda a: a.__class__, unresolved_list))) diff --git a/tests/unit/async_/test_bookmark_manager.py b/tests/unit/async_/test_bookmark_manager.py new file mode 100644 index 000000000..24294b9e5 --- /dev/null +++ b/tests/unit/async_/test_bookmark_manager.py @@ -0,0 +1,295 @@ +# Copyright (c) "Neo4j" +# Neo4j Sweden AB [https://neo4j.com] +# +# This file is part of Neo4j. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import annotations + +import itertools +import typing as t + +import pytest + +import neo4j +from neo4j._async.bookmark_manager import AsyncNeo4jBookmarkManager +from neo4j._async_compat.util import AsyncUtil +from neo4j.api import Bookmarks + +from ..._async_compat import mark_async_test + + +@pytest.mark.parametrize("db", ("foobar", "system")) +@mark_async_test +async def test_return_empty_if_db_doesnt_exists(db) -> None: + bmm = neo4j.AsyncGraphDatabase.bookmark_manager() + + assert set(await bmm.get_bookmarks(db)) == set() + + +@pytest.mark.parametrize("db", ("db1", "db2", "db3")) +@mark_async_test +async def test_return_initial_bookmarks_for_the_given_db(db) -> None: + initial_bookmarks: t.Dict[str, t.List[str]] = { + "db1": ["db1:bm1", "db1:bm1"], + "db2": [], + "db3": ["db3:bm1", "db3:bm2"], + "db4": ["db4:bm4"] + } + bmm = neo4j.AsyncGraphDatabase.bookmark_manager( + initial_bookmarks=initial_bookmarks + ) + + assert set(await bmm.get_bookmarks(db)) == set(initial_bookmarks[db]) + + +@pytest.mark.parametrize("db", ("db1", "db2", "db3")) +@pytest.mark.parametrize("supplier_async", (True, False)) +@mark_async_test +async def test_return_get_bookmarks_from_bookmark_supplier( + db, mocker, supplier_async +) -> None: + if supplier_async and not AsyncUtil.is_async_code: + pytest.skip("Async only test") + + extra_bookmarks = ["foo:bm1", "bar:bm2", "foo:bm1"] + initial_bookmarks: t.Dict[str, t.List[str]] = { + "db1": ["db1:bm1", "db1:bm1"], + "db2": [], + "db3": ["db3:bm1", "db3:bm2"], + "db4": ["db4:bm4"] + } + mock_cls = mocker.AsyncMock if supplier_async else mocker.Mock + supplier = mock_cls( + return_value=Bookmarks.from_raw_values(extra_bookmarks) + ) + bmm = neo4j.AsyncGraphDatabase.bookmark_manager( + initial_bookmarks=initial_bookmarks, + bookmark_supplier=supplier + ) + + assert set(await bmm.get_bookmarks(db)) == { + *extra_bookmarks, *initial_bookmarks.get(db, []) + } + if supplier_async: + supplier.assert_awaited_once_with(db) + else: + supplier.assert_called_once_with(db) + + +@pytest.mark.parametrize("with_initial_bookmarks", (True, False)) +@mark_async_test +async def test_return_all_bookmarks(with_initial_bookmarks) -> None: + initial_bookmarks: t.Dict[str, t.List[str]] = { + "db1": ["db1:bm1", "db1:bm1"], + "db2": [], + "db3": ["db3:bm1", "db3:bm2"], + "db4": ["db4:bm4"], + "db5": ["db3:bm1"] + } + bmm = neo4j.AsyncGraphDatabase.bookmark_manager( + initial_bookmarks=initial_bookmarks if with_initial_bookmarks else None + ) + + all_bookmarks = await bmm.get_all_bookmarks() + + if with_initial_bookmarks: + assert all_bookmarks == set( + itertools.chain.from_iterable(initial_bookmarks.values()) + ) + else: + assert all_bookmarks == set() + + +@pytest.mark.parametrize("with_initial_bookmarks", (True, False)) +@pytest.mark.parametrize("supplier_async", (True, False)) +@mark_async_test +async def test_return_enriched_bookmarks_list_with_supplied_bookmarks( + with_initial_bookmarks, supplier_async, mocker +) -> None: + if supplier_async and not AsyncUtil.is_async_code: + pytest.skip("Async only test") + + initial_bookmarks: t.Dict[str, t.List[str]] = { + "db1": ["db1:bm1", "db1:bm1"], + "db2": [], + "db3": ["db3:bm1", "db3:bm2"], + "db4": ["db4:bm4"], + } + extra_bookmarks = ["foo:bm1", "bar:bm2", "db3:bm1", "foo:bm1"] + mock_cls = mocker.AsyncMock if supplier_async else mocker.Mock + supplier = mock_cls( + return_value=Bookmarks.from_raw_values(extra_bookmarks) + ) + bmm = neo4j.AsyncGraphDatabase.bookmark_manager( + initial_bookmarks=(initial_bookmarks + if with_initial_bookmarks else None), + bookmark_supplier=supplier + ) + + all_bookmarks = await bmm.get_all_bookmarks() + + if with_initial_bookmarks: + assert all_bookmarks == set( + itertools.chain(*initial_bookmarks.values(), extra_bookmarks) + ) + else: + assert all_bookmarks == set(extra_bookmarks) + if supplier_async: + supplier.assert_awaited_once_with(None) + else: + supplier.assert_called_once_with(None) + + +@mark_async_test +async def test_chains_bookmarks_for_existing_db() -> None: + initial_bookmarks: t.Dict[str, t.List[str]] = { + "db1": ["db1:bm1", "db1:bm1"], + "db2": [], + "db3": ["db3:bm1", "db3:bm2"], + "db4": ["db4:bm4"], + } + bmm = neo4j.AsyncGraphDatabase.bookmark_manager( + initial_bookmarks=initial_bookmarks, + ) + await bmm.update_bookmarks("db3", ["db3:bm1"], ["db3:bm3"]) + new_bookmarks = await bmm.get_bookmarks("db3") + all_bookmarks = await bmm.get_all_bookmarks() + + assert new_bookmarks == {"db3:bm2", "db3:bm3"} + assert all_bookmarks == set( + itertools.chain.from_iterable(initial_bookmarks.values()) + ) - {"db3:bm1"} | {"db3:bm2", "db3:bm3"} + + +@mark_async_test +async def test_add_bookmarks_for_a_non_existing_database() -> None: + initial_bookmarks: t.Dict[str, t.List[str]] = { + "db1": ["db1:bm1", "db1:bm1"], + "db2": [], + "db3": ["db3:bm1", "db3:bm2"], + "db4": ["db4:bm4"], + } + bmm = neo4j.AsyncGraphDatabase.bookmark_manager( + initial_bookmarks=initial_bookmarks, + ) + await bmm.update_bookmarks( + "db5", ["db3:bm1", "db5:bm1"], ["db3:bm3", "db3:bm5"] + ) + new_bookmarks = await bmm.get_bookmarks("db5") + all_bookmarks = await bmm.get_all_bookmarks() + + assert new_bookmarks == {"db3:bm3", "db3:bm5"} + assert all_bookmarks == set( + itertools.chain.from_iterable(initial_bookmarks.values()) + ) | {"db3:bm3", "db3:bm5"} + + +@pytest.mark.parametrize("with_initial_bookmarks", (True, False)) +@pytest.mark.parametrize("consumer_async", (True, False)) +@pytest.mark.parametrize("db", ("db1", "db2", "db3")) +@mark_async_test +async def test_notify_on_new_bookmarks( + with_initial_bookmarks, consumer_async, db, mocker +) -> None: + if consumer_async and not AsyncUtil.is_async_code: + pytest.skip("Async only test") + + initial_bookmarks: t.Dict[str, t.List[str]] = { + "db1": ["db1:bm1", "db1:bm1", "db1:bm2"], + "db2": ["db2:bm1"], + } + mock_cls = mocker.AsyncMock if consumer_async else mocker.Mock + consumer = mock_cls() + bmm = neo4j.AsyncGraphDatabase.bookmark_manager( + initial_bookmarks=(initial_bookmarks + if with_initial_bookmarks else None), + bookmark_consumer=consumer + ) + bookmarks_old = {"db1:bm1", "db3:bm1"} + bookmarks_new = {"db1:bm4"} + await bmm.update_bookmarks(db, bookmarks_old, bookmarks_new) + + if consumer_async: + consumer.assert_awaited_once() + args = consumer.await_args.args + else: + consumer.assert_called_once() + args = consumer.call_args.args + assert args[0] == db + assert isinstance(args[1], Bookmarks) + if with_initial_bookmarks: + expected_bms = ( + set(initial_bookmarks.get(db, [])) - bookmarks_old | bookmarks_new + ) + else: + expected_bms = bookmarks_new + assert args[1].raw_values == expected_bms + + +@pytest.mark.parametrize("consumer_async", (True, False)) +@pytest.mark.parametrize("with_initial_bookmarks", (True, False)) +@pytest.mark.parametrize("db", ("db1", "db2")) +@mark_async_test +async def test_does_not_notify_on_empty_new_bookmark_set( + with_initial_bookmarks, consumer_async, db, mocker +) -> None: + if consumer_async and not AsyncUtil.is_async_code: + pytest.skip("Async only test") + + initial_bookmarks: t.Dict[str, t.List[str]] = { + "db1": ["db1:bm1", "db1:bm2"] + } + mock_cls = mocker.AsyncMock if consumer_async else mocker.Mock + consumer = mock_cls() + bmm = neo4j.AsyncGraphDatabase.bookmark_manager( + initial_bookmarks=(initial_bookmarks + if with_initial_bookmarks else None), + bookmark_consumer=consumer + ) + await bmm.update_bookmarks(db, ["db1:bm1"], []) + + consumer.assert_not_called() + + +@pytest.mark.parametrize("dbs", ( + ["db1"], ["db2"], ["db1", "db2"], ["db1", "db3"], ["db1", "db2", "db3"] +)) +@mark_async_test +async def test_forget_database(dbs) -> None: + initial_bookmarks: t.Dict[str, t.List[str]] = { + "db1": ["db1:bm1", "db1:bm1", "db1:bm2"], + "db2": ["db2:bm1"], + } + bmm = neo4j.AsyncGraphDatabase.bookmark_manager( + initial_bookmarks=initial_bookmarks + ) + + for db in dbs: + assert (await bmm.get_bookmarks(db) + == set(initial_bookmarks.get(db, []))) + + await bmm.forget(dbs) + + # assert the key has been removed (memory optimization) + assert isinstance(bmm, AsyncNeo4jBookmarkManager) + assert (set(bmm._bookmarks.keys()) + == set(initial_bookmarks.keys()) - set(dbs)) + + for db in dbs: + assert await bmm.get_bookmarks(db) == set() + assert await bmm.get_all_bookmarks() == set( + bm for k, v in initial_bookmarks.items() if k not in dbs for bm in v + ) diff --git a/tests/unit/async_/test_driver.py b/tests/unit/async_/test_driver.py index a353857a8..d8664d5db 100644 --- a/tests/unit/async_/test_driver.py +++ b/tests/unit/async_/test_driver.py @@ -16,8 +16,11 @@ # limitations under the License. +from __future__ import annotations + import ssl -from functools import wraps +import typing as t +from contextlib import contextmanager import pytest @@ -34,27 +37,26 @@ ) from neo4j._async_compat.util import AsyncUtil from neo4j.api import ( + AsyncBookmarkManager, + BookmarkManager, READ_ACCESS, WRITE_ACCESS, ) from neo4j.exceptions import ConfigurationError -from ..._async_compat import mark_async_test +from ..._async_compat import ( + AsyncTestDecorators, + mark_async_test, +) -@wraps(AsyncGraphDatabase.driver) -def create_driver(*args, **kwargs): +@contextmanager +def expect_async_experimental_warning(): if AsyncUtil.is_async_code: - with pytest.warns(ExperimentalWarning, match="async") as warnings: - driver = AsyncGraphDatabase.driver(*args, **kwargs) - print(warnings) - return driver + with pytest.warns(ExperimentalWarning, match="async"): + yield else: - return AsyncGraphDatabase.driver(*args, **kwargs) - - -def driver(*args, **kwargs): - return AsyncNeo4jDriver(*args, **kwargs) + yield @pytest.mark.parametrize("protocol", ("bolt://", "bolt+s://", "bolt+ssc://")) @@ -70,7 +72,8 @@ async def test_direct_driver_constructor(protocol, host, port, params, auth_toke with pytest.warns(DeprecationWarning, match="routing context"): driver = AsyncGraphDatabase.driver(uri, auth=auth_token) else: - driver = create_driver(uri, auth=auth_token) + with expect_async_experimental_warning(): + driver = AsyncGraphDatabase.driver(uri, auth=auth_token) assert isinstance(driver, AsyncBoltDriver) await driver.close() @@ -85,7 +88,8 @@ async def test_direct_driver_constructor(protocol, host, port, params, auth_toke @mark_async_test async def test_routing_driver_constructor(protocol, host, port, params, auth_token): uri = protocol + host + port + params - driver = create_driver(uri, auth=auth_token) + with expect_async_experimental_warning(): + driver = AsyncGraphDatabase.driver(uri, auth=auth_token) assert isinstance(driver, AsyncNeo4jDriver) await driver.close() @@ -136,7 +140,7 @@ async def test_routing_driver_constructor(protocol, host, port, params, auth_tok ConfigurationError, "The config settings" ), ( - {"ssl_context": ssl.SSLContext(ssl.PROTOCOL_TLSv1)}, + {"ssl_context": ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)}, ConfigurationError, "The config settings" ), ) @@ -150,7 +154,8 @@ def driver_builder(): with pytest.warns(DeprecationWarning, match="trust"): return AsyncGraphDatabase.driver(test_uri, **test_config) else: - return create_driver(test_uri, **test_config) + with expect_async_experimental_warning(): + return AsyncGraphDatabase.driver(test_uri, **test_config) if "+" in test_uri: # `+s` and `+ssc` are short hand syntax for not having to configure the @@ -169,7 +174,8 @@ def driver_builder(): )) def test_invalid_protocol(test_uri): with pytest.raises(ConfigurationError, match="scheme"): - create_driver(test_uri) + with expect_async_experimental_warning(): + AsyncGraphDatabase.driver(test_uri) @pytest.mark.parametrize( @@ -184,7 +190,8 @@ def test_driver_trust_config_error( test_config, expected_failure, expected_failure_message ): with pytest.raises(expected_failure, match=expected_failure_message): - create_driver("bolt://127.0.0.1:9001", **test_config) + with expect_async_experimental_warning(): + AsyncGraphDatabase.driver("bolt://127.0.0.1:9001", **test_config) @pytest.mark.parametrize("uri", ( @@ -192,28 +199,29 @@ def test_driver_trust_config_error( "neo4j://127.0.0.1:9000", )) @mark_async_test -async def test_driver_opens_write_session_by_default(uri, mocker): - driver = create_driver(uri) +async def test_driver_opens_write_session_by_default(uri, fake_pool, mocker): + with expect_async_experimental_warning(): + driver = AsyncGraphDatabase.driver(uri) from neo4j import AsyncTransaction # we set a specific db, because else the driver would try to fetch a RT # to get hold of the actual home database (which won't work in this # unittest) + driver._pool = fake_pool async with driver.session(database="foobar") as session: - acquire_mock = mocker.patch.object(session._pool, "acquire", - autospec=True) - tx_begin_mock = mocker.patch.object(AsyncTransaction, "_begin", - autospec=True) + # acquire_mock = mocker.patch.object(session._pool, "acquire", + # autospec=True) + tx_mock = mocker.patch("neo4j._async.work.session.AsyncTransaction", + autospec=True) tx = await session.begin_transaction() - acquire_mock.assert_called_once_with( + fake_pool.acquire.assert_awaited_once_with( access_mode=WRITE_ACCESS, timeout=mocker.ANY, database=mocker.ANY, bookmarks=mocker.ANY, liveness_check_timeout=mocker.ANY ) - tx_begin_mock.assert_called_once_with( - tx, + tx._begin.assert_awaited_once_with( mocker.ANY, mocker.ANY, mocker.ANY, @@ -231,7 +239,8 @@ async def test_driver_opens_write_session_by_default(uri, mocker): )) @mark_async_test async def test_verify_connectivity(uri, mocker): - driver = create_driver(uri) + with expect_async_experimental_warning(): + driver = AsyncGraphDatabase.driver(uri) pool_mock = mocker.patch.object(driver, "_pool", autospec=True) try: @@ -258,7 +267,8 @@ async def test_verify_connectivity(uri, mocker): async def test_verify_connectivity_parameters_are_deprecated( uri, kwargs, mocker ): - driver = create_driver(uri) + with expect_async_experimental_warning(): + driver = AsyncGraphDatabase.driver(uri) mocker.patch.object(driver, "_pool", autospec=True) try: @@ -281,7 +291,8 @@ async def test_verify_connectivity_parameters_are_deprecated( async def test_get_server_info_parameters_are_experimental( uri, kwargs, mocker ): - driver = create_driver(uri) + with expect_async_experimental_warning(): + driver = AsyncGraphDatabase.driver(uri) mocker.patch.object(driver, "_pool", autospec=True) try: @@ -289,3 +300,152 @@ async def test_get_server_info_parameters_are_experimental( await driver.get_server_info(**kwargs) finally: await driver.close() + + +@mark_async_test +async def test_with_default_bookmark_manager(mocker) -> None: + bmm = AsyncGraphDatabase.bookmark_manager() + # could be one line, but want to make sure the type checker assigns + # bmm whatever type AsyncGraphDatabase.bookmark_manager() returns + session_cls_mock = mocker.patch("neo4j._async.driver.AsyncSession", + autospec=True) + with expect_async_experimental_warning(): + driver = AsyncGraphDatabase.driver( + "bolt://localhost", bookmark_manager=bmm + ) + async with driver as driver: + _ = driver.session() + session_cls_mock.assert_called_once() + assert session_cls_mock.call_args[0][1].bookmark_manager is bmm + + +@AsyncTestDecorators.mark_async_only_test +async def test_with_custom_inherited_async_bookmark_manager(mocker) -> None: + class BMM(AsyncBookmarkManager): + async def update_bookmarks( + self, database: str, previous_bookmarks: t.Iterable[str], + new_bookmarks: t.Iterable[str] + ) -> None: + ... + + async def get_bookmarks(self, database: str) -> t.Collection[str]: + ... + + async def get_all_bookmarks(self) -> t.Collection[str]: + ... + + async def forget(self, databases: t.Iterable[str]) -> None: + ... + + bmm = BMM() + # could be one line, but want to make sure the type checker assigns + # bmm whatever type AsyncGraphDatabase.bookmark_manager() returns + session_cls_mock = mocker.patch("neo4j._async.driver.AsyncSession", + autospec=True) + with expect_async_experimental_warning(): + driver = AsyncGraphDatabase.driver( + "bolt://localhost", bookmark_manager=bmm + ) + async with driver as driver: + _ = driver.session() + session_cls_mock.assert_called_once() + assert session_cls_mock.call_args[0][1].bookmark_manager is bmm + + +@mark_async_test +async def test_with_custom_inherited_sync_bookmark_manager(mocker) -> None: + class BMM(BookmarkManager): + def update_bookmarks( + self, database: str, previous_bookmarks: t.Iterable[str], + new_bookmarks: t.Iterable[str] + ) -> None: + ... + + def get_bookmarks(self, database: str) -> t.Collection[str]: + ... + + def get_all_bookmarks(self) -> t.Collection[str]: + ... + + def forget(self, databases: t.Iterable[str]) -> None: + ... + + bmm = BMM() + # could be one line, but want to make sure the type checker assigns + # bmm whatever type AsyncGraphDatabase.bookmark_manager() returns + session_cls_mock = mocker.patch("neo4j._async.driver.AsyncSession", + autospec=True) + with expect_async_experimental_warning(): + driver = AsyncGraphDatabase.driver( + "bolt://localhost", bookmark_manager=bmm + ) + async with driver as driver: + _ = driver.session() + session_cls_mock.assert_called_once() + assert session_cls_mock.call_args[0][1].bookmark_manager is bmm + + +@AsyncTestDecorators.mark_async_only_test +async def test_with_custom_ducktype_async_bookmark_manager(mocker) -> None: + class BMM: + async def update_bookmarks( + self, database: str, previous_bookmarks: t.Iterable[str], + new_bookmarks: t.Iterable[str] + ) -> None: + ... + + async def get_bookmarks(self, database: str) -> t.Collection[str]: + ... + + async def get_all_bookmarks(self) -> t.Collection[str]: + ... + + async def forget(self, databases: t.Iterable[str]) -> None: + ... + + bmm = BMM() + # could be one line, but want to make sure the type checker assigns + # bmm whatever type AsyncGraphDatabase.bookmark_manager() returns + session_cls_mock = mocker.patch("neo4j._async.driver.AsyncSession", + autospec=True) + with expect_async_experimental_warning(): + driver = AsyncGraphDatabase.driver( + "bolt://localhost", bookmark_manager=bmm + ) + async with driver as driver: + _ = driver.session() + session_cls_mock.assert_called_once() + assert session_cls_mock.call_args[0][1].bookmark_manager is bmm + + +@mark_async_test +async def test_with_custom_ducktype_sync_bookmark_manager(mocker) -> None: + class BMM: + def update_bookmarks( + self, database: str, previous_bookmarks: t.Iterable[str], + new_bookmarks: t.Iterable[str] + ) -> None: + ... + + def get_bookmarks(self, database: str) -> t.Collection[str]: + ... + + def get_all_bookmarks(self) -> t.Collection[str]: + ... + + def forget(self, databases: t.Iterable[str]) -> None: + ... + + bmm = BMM() + # could be one line, but want to make sure the type checker assigns + # bmm whatever type AsyncGraphDatabase.bookmark_manager() returns + session_cls_mock = mocker.patch("neo4j._async.driver.AsyncSession", + autospec=True) + with expect_async_experimental_warning(): + driver = AsyncGraphDatabase.driver( + "bolt://localhost", bookmark_manager=bmm + ) + async with driver as driver: + _ = driver.session() + session_cls_mock.assert_called_once() + assert session_cls_mock.call_args[0][1].bookmark_manager is bmm diff --git a/tests/unit/async_/work/__init__.py b/tests/unit/async_/work/__init__.py index d5ea8d85c..c42cc6fb6 100644 --- a/tests/unit/async_/work/__init__.py +++ b/tests/unit/async_/work/__init__.py @@ -14,9 +14,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - - -from ._fake_connection import ( - async_fake_connection, - async_fake_connection_generator, -) diff --git a/tests/unit/async_/work/conftest.py b/tests/unit/async_/work/conftest.py deleted file mode 100644 index 3b60f3efd..000000000 --- a/tests/unit/async_/work/conftest.py +++ /dev/null @@ -1,6 +0,0 @@ -from ._fake_connection import ( - async_fake_connection, - async_fake_connection_generator, - async_scripted_connection, - async_scripted_connection_generator, -) diff --git a/tests/unit/async_/work/test_session.py b/tests/unit/async_/work/test_session.py index 117cb5cbf..8723cf4c4 100644 --- a/tests/unit/async_/work/test_session.py +++ b/tests/unit/async_/work/test_session.py @@ -27,27 +27,16 @@ Bookmarks, unit_of_work, ) -from neo4j._async.io._pool import AsyncIOPool +from neo4j._async.io import ( + AsyncBoltPool, + AsyncNeo4jPool, +) from neo4j._conf import SessionConfig +from neo4j.api import AsyncBookmarkManager from ...._async_compat import mark_async_test -@pytest.fixture() -def pool(async_fake_connection_generator, mocker): - pool = mocker.AsyncMock(spec=AsyncIOPool) - assert not hasattr(pool, "acquired_connection_mocks") - pool.acquired_connection_mocks = [] - - def acquire_side_effect(*_, **__): - connection = async_fake_connection_generator() - pool.acquired_connection_mocks.append(connection) - return connection - - pool.acquire.side_effect = acquire_side_effect - return pool - - @mark_async_test async def test_session_context_calls_close(mocker): s = AsyncSession(None, SessionConfig()) @@ -66,9 +55,9 @@ async def test_session_context_calls_close(mocker): )) @mark_async_test async def test_opens_connection_on_run( - pool, test_run_args, repetitions, consume + fake_pool, test_run_args, repetitions, consume ): - async with AsyncSession(pool, SessionConfig()) as session: + async with AsyncSession(fake_pool, SessionConfig()) as session: assert session._connection is None result = await session.run(*test_run_args) assert session._connection is not None @@ -82,9 +71,9 @@ async def test_opens_connection_on_run( @pytest.mark.parametrize("repetitions", range(1, 3)) @mark_async_test async def test_closes_connection_after_consume( - pool, test_run_args, repetitions + fake_pool, test_run_args, repetitions ): - async with AsyncSession(pool, SessionConfig()) as session: + async with AsyncSession(fake_pool, SessionConfig()) as session: result = await session.run(*test_run_args) await result.consume() assert session._connection is None @@ -96,9 +85,9 @@ async def test_closes_connection_after_consume( )) @mark_async_test async def test_keeps_connection_until_last_result_consumed( - pool, test_run_args + fake_pool, test_run_args ): - async with AsyncSession(pool, SessionConfig()) as session: + async with AsyncSession(fake_pool, SessionConfig()) as session: result1 = await session.run(*test_run_args) result2 = await session.run(*test_run_args) assert session._connection is not None @@ -109,8 +98,8 @@ async def test_keeps_connection_until_last_result_consumed( @mark_async_test -async def test_opens_connection_on_tx_begin(pool): - async with AsyncSession(pool, SessionConfig()) as session: +async def test_opens_connection_on_tx_begin(fake_pool): + async with AsyncSession(fake_pool, SessionConfig()) as session: assert session._connection is None async with await session.begin_transaction() as _: assert session._connection is not None @@ -121,8 +110,10 @@ async def test_opens_connection_on_tx_begin(pool): )) @pytest.mark.parametrize("repetitions", range(1, 3)) @mark_async_test -async def test_keeps_connection_on_tx_run(pool, test_run_args, repetitions): - async with AsyncSession(pool, SessionConfig()) as session: +async def test_keeps_connection_on_tx_run( + fake_pool, test_run_args, repetitions +): + async with AsyncSession(fake_pool, SessionConfig()) as session: async with await session.begin_transaction() as tx: for _ in range(repetitions): await tx.run(*test_run_args) @@ -135,9 +126,9 @@ async def test_keeps_connection_on_tx_run(pool, test_run_args, repetitions): @pytest.mark.parametrize("repetitions", range(1, 3)) @mark_async_test async def test_keeps_connection_on_tx_consume( - pool, test_run_args, repetitions + fake_pool, test_run_args, repetitions ): - async with AsyncSession(pool, SessionConfig()) as session: + async with AsyncSession(fake_pool, SessionConfig()) as session: async with await session.begin_transaction() as tx: for _ in range(repetitions): result = await tx.run(*test_run_args) @@ -149,8 +140,8 @@ async def test_keeps_connection_on_tx_consume( ("RETURN $x", {"x": 1}), ("RETURN 1",) )) @mark_async_test -async def test_closes_connection_after_tx_close(pool, test_run_args): - async with AsyncSession(pool, SessionConfig()) as session: +async def test_closes_connection_after_tx_close(fake_pool, test_run_args): + async with AsyncSession(fake_pool, SessionConfig()) as session: async with await session.begin_transaction() as tx: for _ in range(2): result = await tx.run(*test_run_args) @@ -164,8 +155,8 @@ async def test_closes_connection_after_tx_close(pool, test_run_args): ("RETURN $x", {"x": 1}), ("RETURN 1",) )) @mark_async_test -async def test_closes_connection_after_tx_commit(pool, test_run_args): - async with AsyncSession(pool, SessionConfig()) as session: +async def test_closes_connection_after_tx_commit(fake_pool, test_run_args): + async with AsyncSession(fake_pool, SessionConfig()) as session: async with await session.begin_transaction() as tx: for _ in range(2): result = await tx.run(*test_run_args) @@ -180,13 +171,13 @@ async def test_closes_connection_after_tx_commit(pool, test_run_args): (None, [], ["abc"], ["foo", "bar"], {"a", "b"}, ("1", "two")) ) @mark_async_test -async def test_session_returns_bookmarks_directly(pool, bookmark_values): +async def test_session_returns_bookmarks_directly(fake_pool, bookmark_values): if bookmark_values is not None: bookmarks = Bookmarks.from_raw_values(bookmark_values) else: bookmarks = Bookmarks() async with AsyncSession( - pool, SessionConfig(bookmarks=bookmarks) + fake_pool, SessionConfig(bookmarks=bookmarks) ) as session: ret_bookmarks = (await session.last_bookmarks()) assert isinstance(ret_bookmarks, Bookmarks) @@ -202,12 +193,13 @@ async def test_session_returns_bookmarks_directly(pool, bookmark_values): (None, [], ["abc"], ["foo", "bar"], ("1", "two")) ) @mark_async_test -async def test_session_last_bookmark_is_deprecated(pool, bookmarks): +async def test_session_last_bookmark_is_deprecated(fake_pool, bookmarks): if bookmarks is not None: with pytest.warns(DeprecationWarning): - session = AsyncSession(pool, SessionConfig(bookmarks=bookmarks)) + session = AsyncSession(fake_pool, + SessionConfig(bookmarks=bookmarks)) else: - session = AsyncSession(pool, SessionConfig(bookmarks=bookmarks)) + session = AsyncSession(fake_pool, SessionConfig(bookmarks=bookmarks)) async with session: with pytest.warns(DeprecationWarning): if bookmarks: @@ -221,9 +213,11 @@ async def test_session_last_bookmark_is_deprecated(pool, bookmarks): (("foo",), ("foo", "bar"), (), ["foo", "bar"], {"a", "b"}) ) @mark_async_test -async def test_session_bookmarks_as_iterable_is_deprecated(pool, bookmarks): +async def test_session_bookmarks_as_iterable_is_deprecated( + fake_pool, bookmarks +): with pytest.warns(DeprecationWarning): - async with AsyncSession(pool, SessionConfig( + async with AsyncSession(fake_pool, SessionConfig( bookmarks=bookmarks )) as session: ret_bookmarks = (await session.last_bookmarks()).raw_values @@ -237,19 +231,19 @@ async def test_session_bookmarks_as_iterable_is_deprecated(pool, bookmarks): (["I don't", "think so"], TypeError), )) @mark_async_test -async def test_session_run_wrong_types(pool, query, error_type): - async with AsyncSession(pool, SessionConfig()) as session: +async def test_session_run_wrong_types(fake_pool, query, error_type): + async with AsyncSession(fake_pool, SessionConfig()) as session: with pytest.raises(error_type): await session.run(query) @pytest.mark.parametrize("tx_type", ("write_transaction", "read_transaction")) @mark_async_test -async def test_tx_function_argument_type(pool, tx_type): +async def test_tx_function_argument_type(fake_pool, tx_type): async def work(tx): assert isinstance(tx, AsyncManagedTransaction) - async with AsyncSession(pool, SessionConfig()) as session: + async with AsyncSession(fake_pool, SessionConfig()) as session: await getattr(session, tx_type)(work) @@ -262,18 +256,20 @@ async def work(tx): )) @mark_async_test -async def test_decorated_tx_function_argument_type(pool, tx_type, decorator_kwargs): +async def test_decorated_tx_function_argument_type( + fake_pool, tx_type, decorator_kwargs +): @unit_of_work(**decorator_kwargs) async def work(tx): assert isinstance(tx, AsyncManagedTransaction) - async with AsyncSession(pool, SessionConfig()) as session: + async with AsyncSession(fake_pool, SessionConfig()) as session: await getattr(session, tx_type)(work) @mark_async_test -async def test_session_tx_type(pool): - async with AsyncSession(pool, SessionConfig()) as session: +async def test_session_tx_type(fake_pool): + async with AsyncSession(fake_pool, SessionConfig()) as session: tx = await session.begin_transaction() assert isinstance(tx, AsyncTransaction) @@ -300,9 +296,9 @@ async def test_session_tx_type(pool): @pytest.mark.parametrize("run_type", ("auto", "unmanaged", "managed")) @mark_async_test async def test_session_run_with_parameters( - pool, parameters, run_type, mocker + fake_pool, parameters, run_type, mocker ): - async with AsyncSession(pool, SessionConfig()) as session: + async with AsyncSession(fake_pool, SessionConfig()) as session: if run_type == "auto": await session.run("RETURN $x", **parameters) elif run_type == "unmanaged": @@ -315,9 +311,133 @@ async def work(tx): else: raise ValueError(run_type) - assert len(pool.acquired_connection_mocks) == 1 - connection_mock = pool.acquired_connection_mocks[0] + assert len(fake_pool.acquired_connection_mocks) == 1 + connection_mock = fake_pool.acquired_connection_mocks[0] assert connection_mock.run.called_once() call = connection_mock.run.call_args assert call.args[0] == "RETURN $x" assert call.kwargs["parameters"] == parameters + + +@pytest.mark.parametrize("db", (None, "adb")) +@pytest.mark.parametrize("routing", (True, False)) +# no home db resolution when connected to Neo4j 4.3 or earlier +@pytest.mark.parametrize("home_db_gets_resolved", (True, False)) +@pytest.mark.parametrize("additional_session_bookmarks", + (None, ["session", "bookmarks"])) +@mark_async_test +async def test_with_bookmark_manager( + fake_pool, db, routing, async_scripted_connection, home_db_gets_resolved, + additional_session_bookmarks, mocker +): + async def update_routing_table_side_effect( + database, imp_user, bookmarks, acquisition_timeout=None, + database_callback=None + ): + if home_db_gets_resolved: + database_callback("homedb") + + async def bmm_get_bookmarks(database): + return [f"{database}:bm1"] + + async def bmm_gat_all_bookmarks(): + return ["all", "bookmarks"] + + async_scripted_connection.set_script([ + ("run", {"on_success": None, "on_summary": None}), + ("pull", { + "on_success": ({"bookmark": "res:bm1", "has_more": False},), + "on_summary": None, + "on_records": None, + }) + ]) + fake_pool.buffered_connection_mocks.append(async_scripted_connection) + + bmm = mocker.Mock(spec=AsyncBookmarkManager) + # res_cls_mock = mocker.patch("neo4j._async.work.session.AsyncResult", + # autospec=True) + bmm.get_bookmarks.side_effect = bmm_get_bookmarks + bmm.get_all_bookmarks.side_effect = bmm_gat_all_bookmarks + + if routing: + fake_pool.mock_add_spec(AsyncNeo4jPool) + fake_pool.update_routing_table.side_effect = \ + update_routing_table_side_effect + else: + fake_pool.mock_add_spec(AsyncBoltPool) + + config = SessionConfig() + config.bookmark_manager = bmm + if db is not None: + config.database = db + if additional_session_bookmarks: + config.bookmarks = Bookmarks.from_raw_values( + additional_session_bookmarks + ) + async with AsyncSession(fake_pool, config) as session: + assert not bmm.method_calls + + await session.run("RETURN 1") + + # assert called bmm accordingly + expected_bmm_method_calls = [mocker.call.get_bookmarks("system"), + mocker.call.get_all_bookmarks()] + if routing and db is None: + expected_bmm_method_calls = [ + # extra call for resolving the home database + mocker.call.get_bookmarks("system"), + *expected_bmm_method_calls + ] + assert bmm.method_calls == expected_bmm_method_calls + assert (bmm.get_bookmarks.await_count + == len(expected_bmm_method_calls) - 1) + bmm.get_all_bookmarks.assert_awaited_once() + bmm.method_calls.clear() + + expected_update_for_db = db + if not db: + if home_db_gets_resolved and routing: + expected_update_for_db = "homedb" + else: + expected_update_for_db = "" + assert [call[0] for call in bmm.method_calls] == ["update_bookmarks"] + assert bmm.method_calls[0].kwargs == {} + assert len(bmm.method_calls[0].args) == 3 + assert bmm.method_calls[0].args[0] == expected_update_for_db + assert (set(bmm.method_calls[0].args[1]) + == {"all", "bookmarks", *(additional_session_bookmarks or [])}) + assert set(bmm.method_calls[0].args[2]) == {"res:bm1"} + + expected_pool_method_calls = ["acquire", "release"] + if routing and db is None: + expected_pool_method_calls = ["update_routing_table", + *expected_pool_method_calls] + assert ([call[0] for call in fake_pool.method_calls] + == expected_pool_method_calls) + assert (set(fake_pool.acquire.call_args.kwargs["bookmarks"]) + == {"system:bm1", *(additional_session_bookmarks or [])}) + if routing and db is None: + assert ( + set(fake_pool.update_routing_table.call_args.kwargs["bookmarks"]) + == {"system:bm1", *(additional_session_bookmarks or [])} + ) + + assert len(fake_pool.acquired_connection_mocks) == 1 + connection_mock = fake_pool.acquired_connection_mocks[0] + assert connection_mock.run.called_once() + connection_run_call_kwargs = connection_mock.run.call_args.kwargs + assert (set(connection_run_call_kwargs["bookmarks"]) + == {"all", "bookmarks", *(additional_session_bookmarks or [])}) + + +@mark_async_test +async def test_with_ignored_bookmark_manager(fake_pool, mocker): + bmm = mocker.Mock(spec=AsyncBookmarkManager) + session_config = SessionConfig() + session_config.bookmark_manager = bmm + session_config.ignore_bookmark_manager = True + async with AsyncSession(fake_pool, session_config) as session: + await session.run("RETURN 1") + + bmm.assert_not_called() + assert not bmm.method_calls diff --git a/tests/unit/common/test_conf.py b/tests/unit/common/test_conf.py index bdda6e41d..408c60433 100644 --- a/tests/unit/common/test_conf.py +++ b/tests/unit/common/test_conf.py @@ -67,6 +67,7 @@ "impersonated_user": None, "fetch_size": 100, "bookmark_manager": object(), + "ignore_bookmark_manager": False, } diff --git a/tests/unit/sync/conftest.py b/tests/unit/sync/conftest.py new file mode 100644 index 000000000..9d171987d --- /dev/null +++ b/tests/unit/sync/conftest.py @@ -0,0 +1,19 @@ +# Copyright (c) "Neo4j" +# Neo4j Sweden AB [https://neo4j.com] +# +# This file is part of Neo4j. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .fixtures import * # necessary for pytest to discover the fixtures diff --git a/tests/unit/sync/fixtures/__init__.py b/tests/unit/sync/fixtures/__init__.py new file mode 100644 index 000000000..c3e907d44 --- /dev/null +++ b/tests/unit/sync/fixtures/__init__.py @@ -0,0 +1,20 @@ +# Copyright (c) "Neo4j" +# Neo4j Sweden AB [https://neo4j.com] +# +# This file is part of Neo4j. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .fake_connection import * +from .fake_pool import * diff --git a/tests/unit/sync/fixtures/fake_connection.py b/tests/unit/sync/fixtures/fake_connection.py new file mode 100644 index 000000000..f3a8e695a --- /dev/null +++ b/tests/unit/sync/fixtures/fake_connection.py @@ -0,0 +1,215 @@ +# Copyright (c) "Neo4j" +# Neo4j Sweden AB [https://neo4j.com] +# +# This file is part of Neo4j. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import inspect + +import pytest + +from neo4j import ServerInfo +from neo4j._deadline import Deadline +from neo4j._sync.io import Bolt + + +__all__ = [ + "fake_connection_generator", + "fake_connection", + "scripted_connection_generator", + "scripted_connection", +] + + +@pytest.fixture +def fake_connection_generator(session_mocker): + mock = session_mocker.mock_module + + class FakeConnection(mock.NonCallableMagicMock): + callbacks = [] + server_info = ServerInfo("127.0.0.1", (4, 3)) + local_port = 1234 + + def __init__(self, *args, **kwargs): + kwargs["spec"] = Bolt + super().__init__(*args, **kwargs) + self.attach_mock(mock.Mock(return_value=True), "is_reset_mock") + self.attach_mock(mock.Mock(return_value=False), "defunct") + self.attach_mock(mock.Mock(return_value=False), "stale") + self.attach_mock(mock.Mock(return_value=False), "closed") + self.attach_mock(mock.Mock(return_value=False), "socket") + self.attach_mock(mock.Mock(), "unresolved_address") + + def close_side_effect(): + self.closed.return_value = True + + self.attach_mock(mock.Mock(side_effect=close_side_effect), + "close") + + self.socket.attach_mock( + mock.Mock(return_value=None), "get_deadline" + ) + + def set_deadline_side_effect(deadline): + deadline = Deadline.from_timeout_or_deadline(deadline) + self.socket.get_deadline.return_value = deadline + + self.socket.attach_mock( + mock.Mock(side_effect=set_deadline_side_effect), "set_deadline" + ) + + @property + def is_reset(self): + if self.closed.return_value or self.defunct.return_value: + raise AssertionError( + "is_reset should not be called on a closed or defunct " + "connection." + ) + return self.is_reset_mock() + + def fetch_message(self, *args, **kwargs): + if self.callbacks: + cb = self.callbacks.pop(0) + cb() + return super().__getattr__("fetch_message")(*args, **kwargs) + + def fetch_all(self, *args, **kwargs): + while self.callbacks: + cb = self.callbacks.pop(0) + cb() + return super().__getattr__("fetch_all")(*args, **kwargs) + + def __getattr__(self, name): + parent = super() + + def build_message_handler(name): + def func(*args, **kwargs): + def callback(): + for cb_name, param_count in ( + ("on_success", 1), + ("on_summary", 0) + ): + cb = kwargs.get(cb_name, None) + if callable(cb): + try: + param_count = \ + len(inspect.signature(cb).parameters) + except ValueError: + # e.g. built-in method as cb + pass + if param_count == 1: + res = cb({}) + else: + res = cb() + try: + res # maybe the callback is async + except TypeError: + pass # or maybe it wasn't ;) + + self.callbacks.append(callback) + + return func + + method_mock = parent.__getattr__(name) + if name in ("run", "commit", "pull", "rollback", "discard"): + method_mock.side_effect = build_message_handler(name) + return method_mock + + return FakeConnection + + +@pytest.fixture +def fake_connection(fake_connection_generator): + return fake_connection_generator() + + +@pytest.fixture +def scripted_connection_generator(fake_connection_generator): + class ScriptedConnection(fake_connection_generator): + _script = [] + _script_pos = 0 + + def set_script(self, callbacks): + """Set a scripted sequence of callbacks. + + :param callbacks: The callbacks. They should be a list of 2-tuples. + `("name_of_message", {"callback_name": arguments})`. E.g., + ``` + [ + ("run", {"on_success": ({},), "on_summary": None}), + ("pull", { + "on_success": None, + "on_summary": None, + "on_records": + }) + ] + ``` + Note that arguments can be `None`. In this case, ScriptedConnection + will make a guess on best-suited default arguments. + """ + self._script = callbacks + self._script_pos = 0 + + def __getattr__(self, name): + parent = super() + + def build_message_handler(name): + def func(*args, **kwargs): + try: + expected_message, scripted_callbacks = \ + self._script[self._script_pos] + except IndexError: + pytest.fail("End of scripted connection reached.") + assert name == expected_message + self._script_pos += 1 + + def callback(): + for cb_name, default_cb_args in ( + ("on_ignored", ({},)), + ("on_failure", ({},)), + ("on_records", ([],)), + ("on_success", ({},)), + ("on_summary", ()), + ): + cb = kwargs.get(cb_name, None) + if ( + not callable(cb) + or cb_name not in scripted_callbacks + ): + continue + cb_args = scripted_callbacks[cb_name] + if cb_args is None: + cb_args = default_cb_args + res = cb(*cb_args) + try: + res # maybe the callback is async + except TypeError: + pass # or maybe it wasn't ;) + + self.callbacks.append(callback) + + return func + + method_mock = parent.__getattr__(name) + if name in ("run", "commit", "pull", "rollback", "discard"): + method_mock.side_effect = build_message_handler(name) + return method_mock + + return ScriptedConnection + + +@pytest.fixture +def scripted_connection(scripted_connection_generator): + return scripted_connection_generator() diff --git a/tests/unit/sync/fixtures/fake_pool.py b/tests/unit/sync/fixtures/fake_pool.py new file mode 100644 index 000000000..63f5853ed --- /dev/null +++ b/tests/unit/sync/fixtures/fake_pool.py @@ -0,0 +1,45 @@ +# Copyright (c) "Neo4j" +# Neo4j Sweden AB [https://neo4j.com] +# +# This file is part of Neo4j. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytest + +from neo4j._sync.io._pool import IOPool + + +__all__ = [ + "fake_pool", +] + + +@pytest.fixture +def fake_pool(fake_connection_generator, mocker): + pool = mocker.Mock(spec=IOPool) + assert not hasattr(pool, "acquired_connection_mocks") + pool.buffered_connection_mocks = [] + pool.acquired_connection_mocks = [] + + def acquire_side_effect(*_, **__): + if pool.buffered_connection_mocks: + connection = pool.buffered_connection_mocks.pop() + else: + connection = fake_connection_generator() + pool.acquired_connection_mocks.append(connection) + return connection + + pool.acquire.side_effect = acquire_side_effect + return pool diff --git a/tests/unit/sync/io/test_neo4j_pool.py b/tests/unit/sync/io/test_neo4j_pool.py index 6b9f8d7eb..bca9d4441 100644 --- a/tests/unit/sync/io/test_neo4j_pool.py +++ b/tests/unit/sync/io/test_neo4j_pool.py @@ -38,7 +38,6 @@ ) from ...._async_compat import mark_sync_test -from ..work import fake_connection_generator # needed as fixture ROUTER_ADDRESS = ResolvedAddress(("1.2.3.1", 9001), host_name="host") diff --git a/tests/unit/sync/test_addressing.py b/tests/unit/sync/test_addressing.py index 2779eb8bf..444b289d4 100644 --- a/tests/unit/sync/test_addressing.py +++ b/tests/unit/sync/test_addressing.py @@ -34,7 +34,7 @@ @mark_sync_test -def test_address_resolve(): +def test_address_resolve() -> None: address = Address(("127.0.0.1", 7687)) resolved = NetworkUtil.resolve_address(address) resolved = Util.list(resolved) @@ -45,7 +45,7 @@ def test_address_resolve(): @mark_sync_test -def test_address_resolve_with_custom_resolver_none(): +def test_address_resolve_with_custom_resolver_none() -> None: address = Address(("127.0.0.1", 7687)) resolved = NetworkUtil.resolve_address(address, resolver=None) resolved = Util.list(resolved) @@ -64,7 +64,9 @@ def test_address_resolve_with_custom_resolver_none(): ) @mark_sync_test -def test_address_resolve_with_unresolvable_address(test_input, expected): +def test_address_resolve_with_unresolvable_address( + test_input, expected +) -> None: with pytest.raises(expected): Util.list( NetworkUtil.resolve_address(test_input, resolver=None) @@ -73,7 +75,7 @@ def test_address_resolve_with_unresolvable_address(test_input, expected): @mark_sync_test @pytest.mark.parametrize("resolver_type", ("sync", "async")) -def test_address_resolve_with_custom_resolver(resolver_type): +def test_address_resolve_with_custom_resolver(resolver_type) -> None: def custom_resolver_sync(_): return [("127.0.0.1", 7687), ("localhost", 1234)] @@ -98,9 +100,11 @@ def custom_resolver_async(_): @mark_sync_test -def test_address_unresolve(): +def test_address_unresolve() -> None: + def custom_resolver(_): + return custom_resolved + custom_resolved = [("127.0.0.1", 7687), ("localhost", 4321)] - custom_resolver = lambda _: custom_resolved address = Address(("foobar", 1234)) unresolved = address.unresolved @@ -109,9 +113,9 @@ def test_address_unresolve(): resolved = NetworkUtil.resolve_address( address, family=AF_INET, resolver=custom_resolver ) - resolved = Util.list(resolved) - custom_resolved = sorted(Address(a) for a in custom_resolved) - unresolved = sorted(a.unresolved for a in resolved) - assert custom_resolved == unresolved - assert (list(map(lambda a: a.__class__, custom_resolved)) - == list(map(lambda a: a.__class__, unresolved))) + resolved_list = Util.list(resolved) + custom_resolved_addresses = sorted(Address(a) for a in custom_resolved) + unresolved_list = sorted(a.unresolved for a in resolved_list) + assert custom_resolved_addresses == unresolved_list + assert (list(map(lambda a: a.__class__, custom_resolved_addresses)) + == list(map(lambda a: a.__class__, unresolved_list))) diff --git a/tests/unit/sync/test_bookmark_manager.py b/tests/unit/sync/test_bookmark_manager.py new file mode 100644 index 000000000..eab8f19ef --- /dev/null +++ b/tests/unit/sync/test_bookmark_manager.py @@ -0,0 +1,295 @@ +# Copyright (c) "Neo4j" +# Neo4j Sweden AB [https://neo4j.com] +# +# This file is part of Neo4j. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import annotations + +import itertools +import typing as t + +import pytest + +import neo4j +from neo4j._async_compat.util import Util +from neo4j._sync.bookmark_manager import Neo4jBookmarkManager +from neo4j.api import Bookmarks + +from ..._async_compat import mark_sync_test + + +@pytest.mark.parametrize("db", ("foobar", "system")) +@mark_sync_test +def test_return_empty_if_db_doesnt_exists(db) -> None: + bmm = neo4j.GraphDatabase.bookmark_manager() + + assert set(bmm.get_bookmarks(db)) == set() + + +@pytest.mark.parametrize("db", ("db1", "db2", "db3")) +@mark_sync_test +def test_return_initial_bookmarks_for_the_given_db(db) -> None: + initial_bookmarks: t.Dict[str, t.List[str]] = { + "db1": ["db1:bm1", "db1:bm1"], + "db2": [], + "db3": ["db3:bm1", "db3:bm2"], + "db4": ["db4:bm4"] + } + bmm = neo4j.GraphDatabase.bookmark_manager( + initial_bookmarks=initial_bookmarks + ) + + assert set(bmm.get_bookmarks(db)) == set(initial_bookmarks[db]) + + +@pytest.mark.parametrize("db", ("db1", "db2", "db3")) +@pytest.mark.parametrize("supplier_async", (True, False)) +@mark_sync_test +def test_return_get_bookmarks_from_bookmark_supplier( + db, mocker, supplier_async +) -> None: + if supplier_async and not Util.is_async_code: + pytest.skip("Async only test") + + extra_bookmarks = ["foo:bm1", "bar:bm2", "foo:bm1"] + initial_bookmarks: t.Dict[str, t.List[str]] = { + "db1": ["db1:bm1", "db1:bm1"], + "db2": [], + "db3": ["db3:bm1", "db3:bm2"], + "db4": ["db4:bm4"] + } + mock_cls = mocker.Mock if supplier_async else mocker.Mock + supplier = mock_cls( + return_value=Bookmarks.from_raw_values(extra_bookmarks) + ) + bmm = neo4j.GraphDatabase.bookmark_manager( + initial_bookmarks=initial_bookmarks, + bookmark_supplier=supplier + ) + + assert set(bmm.get_bookmarks(db)) == { + *extra_bookmarks, *initial_bookmarks.get(db, []) + } + if supplier_async: + supplier.assert_called_once_with(db) + else: + supplier.assert_called_once_with(db) + + +@pytest.mark.parametrize("with_initial_bookmarks", (True, False)) +@mark_sync_test +def test_return_all_bookmarks(with_initial_bookmarks) -> None: + initial_bookmarks: t.Dict[str, t.List[str]] = { + "db1": ["db1:bm1", "db1:bm1"], + "db2": [], + "db3": ["db3:bm1", "db3:bm2"], + "db4": ["db4:bm4"], + "db5": ["db3:bm1"] + } + bmm = neo4j.GraphDatabase.bookmark_manager( + initial_bookmarks=initial_bookmarks if with_initial_bookmarks else None + ) + + all_bookmarks = bmm.get_all_bookmarks() + + if with_initial_bookmarks: + assert all_bookmarks == set( + itertools.chain.from_iterable(initial_bookmarks.values()) + ) + else: + assert all_bookmarks == set() + + +@pytest.mark.parametrize("with_initial_bookmarks", (True, False)) +@pytest.mark.parametrize("supplier_async", (True, False)) +@mark_sync_test +def test_return_enriched_bookmarks_list_with_supplied_bookmarks( + with_initial_bookmarks, supplier_async, mocker +) -> None: + if supplier_async and not Util.is_async_code: + pytest.skip("Async only test") + + initial_bookmarks: t.Dict[str, t.List[str]] = { + "db1": ["db1:bm1", "db1:bm1"], + "db2": [], + "db3": ["db3:bm1", "db3:bm2"], + "db4": ["db4:bm4"], + } + extra_bookmarks = ["foo:bm1", "bar:bm2", "db3:bm1", "foo:bm1"] + mock_cls = mocker.Mock if supplier_async else mocker.Mock + supplier = mock_cls( + return_value=Bookmarks.from_raw_values(extra_bookmarks) + ) + bmm = neo4j.GraphDatabase.bookmark_manager( + initial_bookmarks=(initial_bookmarks + if with_initial_bookmarks else None), + bookmark_supplier=supplier + ) + + all_bookmarks = bmm.get_all_bookmarks() + + if with_initial_bookmarks: + assert all_bookmarks == set( + itertools.chain(*initial_bookmarks.values(), extra_bookmarks) + ) + else: + assert all_bookmarks == set(extra_bookmarks) + if supplier_async: + supplier.assert_called_once_with(None) + else: + supplier.assert_called_once_with(None) + + +@mark_sync_test +def test_chains_bookmarks_for_existing_db() -> None: + initial_bookmarks: t.Dict[str, t.List[str]] = { + "db1": ["db1:bm1", "db1:bm1"], + "db2": [], + "db3": ["db3:bm1", "db3:bm2"], + "db4": ["db4:bm4"], + } + bmm = neo4j.GraphDatabase.bookmark_manager( + initial_bookmarks=initial_bookmarks, + ) + bmm.update_bookmarks("db3", ["db3:bm1"], ["db3:bm3"]) + new_bookmarks = bmm.get_bookmarks("db3") + all_bookmarks = bmm.get_all_bookmarks() + + assert new_bookmarks == {"db3:bm2", "db3:bm3"} + assert all_bookmarks == set( + itertools.chain.from_iterable(initial_bookmarks.values()) + ) - {"db3:bm1"} | {"db3:bm2", "db3:bm3"} + + +@mark_sync_test +def test_add_bookmarks_for_a_non_existing_database() -> None: + initial_bookmarks: t.Dict[str, t.List[str]] = { + "db1": ["db1:bm1", "db1:bm1"], + "db2": [], + "db3": ["db3:bm1", "db3:bm2"], + "db4": ["db4:bm4"], + } + bmm = neo4j.GraphDatabase.bookmark_manager( + initial_bookmarks=initial_bookmarks, + ) + bmm.update_bookmarks( + "db5", ["db3:bm1", "db5:bm1"], ["db3:bm3", "db3:bm5"] + ) + new_bookmarks = bmm.get_bookmarks("db5") + all_bookmarks = bmm.get_all_bookmarks() + + assert new_bookmarks == {"db3:bm3", "db3:bm5"} + assert all_bookmarks == set( + itertools.chain.from_iterable(initial_bookmarks.values()) + ) | {"db3:bm3", "db3:bm5"} + + +@pytest.mark.parametrize("with_initial_bookmarks", (True, False)) +@pytest.mark.parametrize("consumer_async", (True, False)) +@pytest.mark.parametrize("db", ("db1", "db2", "db3")) +@mark_sync_test +def test_notify_on_new_bookmarks( + with_initial_bookmarks, consumer_async, db, mocker +) -> None: + if consumer_async and not Util.is_async_code: + pytest.skip("Async only test") + + initial_bookmarks: t.Dict[str, t.List[str]] = { + "db1": ["db1:bm1", "db1:bm1", "db1:bm2"], + "db2": ["db2:bm1"], + } + mock_cls = mocker.Mock if consumer_async else mocker.Mock + consumer = mock_cls() + bmm = neo4j.GraphDatabase.bookmark_manager( + initial_bookmarks=(initial_bookmarks + if with_initial_bookmarks else None), + bookmark_consumer=consumer + ) + bookmarks_old = {"db1:bm1", "db3:bm1"} + bookmarks_new = {"db1:bm4"} + bmm.update_bookmarks(db, bookmarks_old, bookmarks_new) + + if consumer_async: + consumer.assert_called_once() + args = consumer.await_args.args + else: + consumer.assert_called_once() + args = consumer.call_args.args + assert args[0] == db + assert isinstance(args[1], Bookmarks) + if with_initial_bookmarks: + expected_bms = ( + set(initial_bookmarks.get(db, [])) - bookmarks_old | bookmarks_new + ) + else: + expected_bms = bookmarks_new + assert args[1].raw_values == expected_bms + + +@pytest.mark.parametrize("consumer_async", (True, False)) +@pytest.mark.parametrize("with_initial_bookmarks", (True, False)) +@pytest.mark.parametrize("db", ("db1", "db2")) +@mark_sync_test +def test_does_not_notify_on_empty_new_bookmark_set( + with_initial_bookmarks, consumer_async, db, mocker +) -> None: + if consumer_async and not Util.is_async_code: + pytest.skip("Async only test") + + initial_bookmarks: t.Dict[str, t.List[str]] = { + "db1": ["db1:bm1", "db1:bm2"] + } + mock_cls = mocker.Mock if consumer_async else mocker.Mock + consumer = mock_cls() + bmm = neo4j.GraphDatabase.bookmark_manager( + initial_bookmarks=(initial_bookmarks + if with_initial_bookmarks else None), + bookmark_consumer=consumer + ) + bmm.update_bookmarks(db, ["db1:bm1"], []) + + consumer.assert_not_called() + + +@pytest.mark.parametrize("dbs", ( + ["db1"], ["db2"], ["db1", "db2"], ["db1", "db3"], ["db1", "db2", "db3"] +)) +@mark_sync_test +def test_forget_database(dbs) -> None: + initial_bookmarks: t.Dict[str, t.List[str]] = { + "db1": ["db1:bm1", "db1:bm1", "db1:bm2"], + "db2": ["db2:bm1"], + } + bmm = neo4j.GraphDatabase.bookmark_manager( + initial_bookmarks=initial_bookmarks + ) + + for db in dbs: + assert (bmm.get_bookmarks(db) + == set(initial_bookmarks.get(db, []))) + + bmm.forget(dbs) + + # assert the key has been removed (memory optimization) + assert isinstance(bmm, Neo4jBookmarkManager) + assert (set(bmm._bookmarks.keys()) + == set(initial_bookmarks.keys()) - set(dbs)) + + for db in dbs: + assert bmm.get_bookmarks(db) == set() + assert bmm.get_all_bookmarks() == set( + bm for k, v in initial_bookmarks.items() if k not in dbs for bm in v + ) diff --git a/tests/unit/sync/test_driver.py b/tests/unit/sync/test_driver.py index 0784cc271..f90123399 100644 --- a/tests/unit/sync/test_driver.py +++ b/tests/unit/sync/test_driver.py @@ -16,8 +16,11 @@ # limitations under the License. +from __future__ import annotations + import ssl -from functools import wraps +import typing as t +from contextlib import contextmanager import pytest @@ -34,27 +37,25 @@ ) from neo4j._async_compat.util import Util from neo4j.api import ( + BookmarkManager, READ_ACCESS, WRITE_ACCESS, ) from neo4j.exceptions import ConfigurationError -from ..._async_compat import mark_sync_test +from ..._async_compat import ( + mark_sync_test, + TestDecorators, +) -@wraps(GraphDatabase.driver) -def create_driver(*args, **kwargs): +@contextmanager +def expect_async_experimental_warning(): if Util.is_async_code: - with pytest.warns(ExperimentalWarning, match="async") as warnings: - driver = GraphDatabase.driver(*args, **kwargs) - print(warnings) - return driver + with pytest.warns(ExperimentalWarning, match="async"): + yield else: - return GraphDatabase.driver(*args, **kwargs) - - -def driver(*args, **kwargs): - return Neo4jDriver(*args, **kwargs) + yield @pytest.mark.parametrize("protocol", ("bolt://", "bolt+s://", "bolt+ssc://")) @@ -70,7 +71,8 @@ def test_direct_driver_constructor(protocol, host, port, params, auth_token): with pytest.warns(DeprecationWarning, match="routing context"): driver = GraphDatabase.driver(uri, auth=auth_token) else: - driver = create_driver(uri, auth=auth_token) + with expect_async_experimental_warning(): + driver = GraphDatabase.driver(uri, auth=auth_token) assert isinstance(driver, BoltDriver) driver.close() @@ -85,7 +87,8 @@ def test_direct_driver_constructor(protocol, host, port, params, auth_token): @mark_sync_test def test_routing_driver_constructor(protocol, host, port, params, auth_token): uri = protocol + host + port + params - driver = create_driver(uri, auth=auth_token) + with expect_async_experimental_warning(): + driver = GraphDatabase.driver(uri, auth=auth_token) assert isinstance(driver, Neo4jDriver) driver.close() @@ -136,7 +139,7 @@ def test_routing_driver_constructor(protocol, host, port, params, auth_token): ConfigurationError, "The config settings" ), ( - {"ssl_context": ssl.SSLContext(ssl.PROTOCOL_TLSv1)}, + {"ssl_context": ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)}, ConfigurationError, "The config settings" ), ) @@ -150,7 +153,8 @@ def driver_builder(): with pytest.warns(DeprecationWarning, match="trust"): return GraphDatabase.driver(test_uri, **test_config) else: - return create_driver(test_uri, **test_config) + with expect_async_experimental_warning(): + return GraphDatabase.driver(test_uri, **test_config) if "+" in test_uri: # `+s` and `+ssc` are short hand syntax for not having to configure the @@ -169,7 +173,8 @@ def driver_builder(): )) def test_invalid_protocol(test_uri): with pytest.raises(ConfigurationError, match="scheme"): - create_driver(test_uri) + with expect_async_experimental_warning(): + GraphDatabase.driver(test_uri) @pytest.mark.parametrize( @@ -184,7 +189,8 @@ def test_driver_trust_config_error( test_config, expected_failure, expected_failure_message ): with pytest.raises(expected_failure, match=expected_failure_message): - create_driver("bolt://127.0.0.1:9001", **test_config) + with expect_async_experimental_warning(): + GraphDatabase.driver("bolt://127.0.0.1:9001", **test_config) @pytest.mark.parametrize("uri", ( @@ -192,28 +198,29 @@ def test_driver_trust_config_error( "neo4j://127.0.0.1:9000", )) @mark_sync_test -def test_driver_opens_write_session_by_default(uri, mocker): - driver = create_driver(uri) +def test_driver_opens_write_session_by_default(uri, fake_pool, mocker): + with expect_async_experimental_warning(): + driver = GraphDatabase.driver(uri) from neo4j import Transaction # we set a specific db, because else the driver would try to fetch a RT # to get hold of the actual home database (which won't work in this # unittest) + driver._pool = fake_pool with driver.session(database="foobar") as session: - acquire_mock = mocker.patch.object(session._pool, "acquire", - autospec=True) - tx_begin_mock = mocker.patch.object(Transaction, "_begin", - autospec=True) + # acquire_mock = mocker.patch.object(session._pool, "acquire", + # autospec=True) + tx_mock = mocker.patch("neo4j._sync.work.session.Transaction", + autospec=True) tx = session.begin_transaction() - acquire_mock.assert_called_once_with( + fake_pool.acquire.assert_called_once_with( access_mode=WRITE_ACCESS, timeout=mocker.ANY, database=mocker.ANY, bookmarks=mocker.ANY, liveness_check_timeout=mocker.ANY ) - tx_begin_mock.assert_called_once_with( - tx, + tx._begin.assert_called_once_with( mocker.ANY, mocker.ANY, mocker.ANY, @@ -231,7 +238,8 @@ def test_driver_opens_write_session_by_default(uri, mocker): )) @mark_sync_test def test_verify_connectivity(uri, mocker): - driver = create_driver(uri) + with expect_async_experimental_warning(): + driver = GraphDatabase.driver(uri) pool_mock = mocker.patch.object(driver, "_pool", autospec=True) try: @@ -258,7 +266,8 @@ def test_verify_connectivity(uri, mocker): def test_verify_connectivity_parameters_are_deprecated( uri, kwargs, mocker ): - driver = create_driver(uri) + with expect_async_experimental_warning(): + driver = GraphDatabase.driver(uri) mocker.patch.object(driver, "_pool", autospec=True) try: @@ -281,7 +290,8 @@ def test_verify_connectivity_parameters_are_deprecated( def test_get_server_info_parameters_are_experimental( uri, kwargs, mocker ): - driver = create_driver(uri) + with expect_async_experimental_warning(): + driver = GraphDatabase.driver(uri) mocker.patch.object(driver, "_pool", autospec=True) try: @@ -289,3 +299,152 @@ def test_get_server_info_parameters_are_experimental( driver.get_server_info(**kwargs) finally: driver.close() + + +@mark_sync_test +def test_with_default_bookmark_manager(mocker) -> None: + bmm = GraphDatabase.bookmark_manager() + # could be one line, but want to make sure the type checker assigns + # bmm whatever type AsyncGraphDatabase.bookmark_manager() returns + session_cls_mock = mocker.patch("neo4j._sync.driver.Session", + autospec=True) + with expect_async_experimental_warning(): + driver = GraphDatabase.driver( + "bolt://localhost", bookmark_manager=bmm + ) + with driver as driver: + _ = driver.session() + session_cls_mock.assert_called_once() + assert session_cls_mock.call_args[0][1].bookmark_manager is bmm + + +@TestDecorators.mark_async_only_test +def test_with_custom_inherited_async_bookmark_manager(mocker) -> None: + class BMM(BookmarkManager): + def update_bookmarks( + self, database: str, previous_bookmarks: t.Iterable[str], + new_bookmarks: t.Iterable[str] + ) -> None: + ... + + def get_bookmarks(self, database: str) -> t.Collection[str]: + ... + + def get_all_bookmarks(self) -> t.Collection[str]: + ... + + def forget(self, databases: t.Iterable[str]) -> None: + ... + + bmm = BMM() + # could be one line, but want to make sure the type checker assigns + # bmm whatever type AsyncGraphDatabase.bookmark_manager() returns + session_cls_mock = mocker.patch("neo4j._sync.driver.Session", + autospec=True) + with expect_async_experimental_warning(): + driver = GraphDatabase.driver( + "bolt://localhost", bookmark_manager=bmm + ) + with driver as driver: + _ = driver.session() + session_cls_mock.assert_called_once() + assert session_cls_mock.call_args[0][1].bookmark_manager is bmm + + +@mark_sync_test +def test_with_custom_inherited_sync_bookmark_manager(mocker) -> None: + class BMM(BookmarkManager): + def update_bookmarks( + self, database: str, previous_bookmarks: t.Iterable[str], + new_bookmarks: t.Iterable[str] + ) -> None: + ... + + def get_bookmarks(self, database: str) -> t.Collection[str]: + ... + + def get_all_bookmarks(self) -> t.Collection[str]: + ... + + def forget(self, databases: t.Iterable[str]) -> None: + ... + + bmm = BMM() + # could be one line, but want to make sure the type checker assigns + # bmm whatever type AsyncGraphDatabase.bookmark_manager() returns + session_cls_mock = mocker.patch("neo4j._sync.driver.Session", + autospec=True) + with expect_async_experimental_warning(): + driver = GraphDatabase.driver( + "bolt://localhost", bookmark_manager=bmm + ) + with driver as driver: + _ = driver.session() + session_cls_mock.assert_called_once() + assert session_cls_mock.call_args[0][1].bookmark_manager is bmm + + +@TestDecorators.mark_async_only_test +def test_with_custom_ducktype_async_bookmark_manager(mocker) -> None: + class BMM: + def update_bookmarks( + self, database: str, previous_bookmarks: t.Iterable[str], + new_bookmarks: t.Iterable[str] + ) -> None: + ... + + def get_bookmarks(self, database: str) -> t.Collection[str]: + ... + + def get_all_bookmarks(self) -> t.Collection[str]: + ... + + def forget(self, databases: t.Iterable[str]) -> None: + ... + + bmm = BMM() + # could be one line, but want to make sure the type checker assigns + # bmm whatever type AsyncGraphDatabase.bookmark_manager() returns + session_cls_mock = mocker.patch("neo4j._sync.driver.Session", + autospec=True) + with expect_async_experimental_warning(): + driver = GraphDatabase.driver( + "bolt://localhost", bookmark_manager=bmm + ) + with driver as driver: + _ = driver.session() + session_cls_mock.assert_called_once() + assert session_cls_mock.call_args[0][1].bookmark_manager is bmm + + +@mark_sync_test +def test_with_custom_ducktype_sync_bookmark_manager(mocker) -> None: + class BMM: + def update_bookmarks( + self, database: str, previous_bookmarks: t.Iterable[str], + new_bookmarks: t.Iterable[str] + ) -> None: + ... + + def get_bookmarks(self, database: str) -> t.Collection[str]: + ... + + def get_all_bookmarks(self) -> t.Collection[str]: + ... + + def forget(self, databases: t.Iterable[str]) -> None: + ... + + bmm = BMM() + # could be one line, but want to make sure the type checker assigns + # bmm whatever type AsyncGraphDatabase.bookmark_manager() returns + session_cls_mock = mocker.patch("neo4j._sync.driver.Session", + autospec=True) + with expect_async_experimental_warning(): + driver = GraphDatabase.driver( + "bolt://localhost", bookmark_manager=bmm + ) + with driver as driver: + _ = driver.session() + session_cls_mock.assert_called_once() + assert session_cls_mock.call_args[0][1].bookmark_manager is bmm diff --git a/tests/unit/sync/work/__init__.py b/tests/unit/sync/work/__init__.py index 27923502c..c42cc6fb6 100644 --- a/tests/unit/sync/work/__init__.py +++ b/tests/unit/sync/work/__init__.py @@ -14,9 +14,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - - -from ._fake_connection import ( - fake_connection, - fake_connection_generator, -) diff --git a/tests/unit/sync/work/_fake_connection.py b/tests/unit/sync/work/_fake_connection.py index 28132a4dc..d55006682 100644 --- a/tests/unit/sync/work/_fake_connection.py +++ b/tests/unit/sync/work/_fake_connection.py @@ -158,15 +158,15 @@ def __getattr__(self, name): parent = super() def build_message_handler(name): - try: - expected_message, scripted_callbacks = \ - self._script[self._script_pos] - except IndexError: - pytest.fail("End of scripted connection reached.") - assert name == expected_message - self._script_pos += 1 - def func(*args, **kwargs): + try: + expected_message, scripted_callbacks = \ + self._script[self._script_pos] + except IndexError: + pytest.fail("End of scripted connection reached.") + assert name == expected_message + self._script_pos += 1 + def callback(): for cb_name, default_cb_args in ( ("on_ignored", ({},)), @@ -176,8 +176,10 @@ def callback(): ("on_summary", ()), ): cb = kwargs.get(cb_name, None) - if (not callable(cb) - or cb_name not in scripted_callbacks): + if ( + not callable(cb) + or cb_name not in scripted_callbacks + ): continue cb_args = scripted_callbacks[cb_name] if cb_args is None: diff --git a/tests/unit/sync/work/test_session.py b/tests/unit/sync/work/test_session.py index c93646306..e1fc1b173 100644 --- a/tests/unit/sync/work/test_session.py +++ b/tests/unit/sync/work/test_session.py @@ -28,26 +28,15 @@ unit_of_work, ) from neo4j._conf import SessionConfig -from neo4j._sync.io._pool import IOPool +from neo4j._sync.io import ( + BoltPool, + Neo4jPool, +) +from neo4j.api import BookmarkManager from ...._async_compat import mark_sync_test -@pytest.fixture() -def pool(fake_connection_generator, mocker): - pool = mocker.Mock(spec=IOPool) - assert not hasattr(pool, "acquired_connection_mocks") - pool.acquired_connection_mocks = [] - - def acquire_side_effect(*_, **__): - connection = fake_connection_generator() - pool.acquired_connection_mocks.append(connection) - return connection - - pool.acquire.side_effect = acquire_side_effect - return pool - - @mark_sync_test def test_session_context_calls_close(mocker): s = Session(None, SessionConfig()) @@ -66,9 +55,9 @@ def test_session_context_calls_close(mocker): )) @mark_sync_test def test_opens_connection_on_run( - pool, test_run_args, repetitions, consume + fake_pool, test_run_args, repetitions, consume ): - with Session(pool, SessionConfig()) as session: + with Session(fake_pool, SessionConfig()) as session: assert session._connection is None result = session.run(*test_run_args) assert session._connection is not None @@ -82,9 +71,9 @@ def test_opens_connection_on_run( @pytest.mark.parametrize("repetitions", range(1, 3)) @mark_sync_test def test_closes_connection_after_consume( - pool, test_run_args, repetitions + fake_pool, test_run_args, repetitions ): - with Session(pool, SessionConfig()) as session: + with Session(fake_pool, SessionConfig()) as session: result = session.run(*test_run_args) result.consume() assert session._connection is None @@ -96,9 +85,9 @@ def test_closes_connection_after_consume( )) @mark_sync_test def test_keeps_connection_until_last_result_consumed( - pool, test_run_args + fake_pool, test_run_args ): - with Session(pool, SessionConfig()) as session: + with Session(fake_pool, SessionConfig()) as session: result1 = session.run(*test_run_args) result2 = session.run(*test_run_args) assert session._connection is not None @@ -109,8 +98,8 @@ def test_keeps_connection_until_last_result_consumed( @mark_sync_test -def test_opens_connection_on_tx_begin(pool): - with Session(pool, SessionConfig()) as session: +def test_opens_connection_on_tx_begin(fake_pool): + with Session(fake_pool, SessionConfig()) as session: assert session._connection is None with session.begin_transaction() as _: assert session._connection is not None @@ -121,8 +110,10 @@ def test_opens_connection_on_tx_begin(pool): )) @pytest.mark.parametrize("repetitions", range(1, 3)) @mark_sync_test -def test_keeps_connection_on_tx_run(pool, test_run_args, repetitions): - with Session(pool, SessionConfig()) as session: +def test_keeps_connection_on_tx_run( + fake_pool, test_run_args, repetitions +): + with Session(fake_pool, SessionConfig()) as session: with session.begin_transaction() as tx: for _ in range(repetitions): tx.run(*test_run_args) @@ -135,9 +126,9 @@ def test_keeps_connection_on_tx_run(pool, test_run_args, repetitions): @pytest.mark.parametrize("repetitions", range(1, 3)) @mark_sync_test def test_keeps_connection_on_tx_consume( - pool, test_run_args, repetitions + fake_pool, test_run_args, repetitions ): - with Session(pool, SessionConfig()) as session: + with Session(fake_pool, SessionConfig()) as session: with session.begin_transaction() as tx: for _ in range(repetitions): result = tx.run(*test_run_args) @@ -149,8 +140,8 @@ def test_keeps_connection_on_tx_consume( ("RETURN $x", {"x": 1}), ("RETURN 1",) )) @mark_sync_test -def test_closes_connection_after_tx_close(pool, test_run_args): - with Session(pool, SessionConfig()) as session: +def test_closes_connection_after_tx_close(fake_pool, test_run_args): + with Session(fake_pool, SessionConfig()) as session: with session.begin_transaction() as tx: for _ in range(2): result = tx.run(*test_run_args) @@ -164,8 +155,8 @@ def test_closes_connection_after_tx_close(pool, test_run_args): ("RETURN $x", {"x": 1}), ("RETURN 1",) )) @mark_sync_test -def test_closes_connection_after_tx_commit(pool, test_run_args): - with Session(pool, SessionConfig()) as session: +def test_closes_connection_after_tx_commit(fake_pool, test_run_args): + with Session(fake_pool, SessionConfig()) as session: with session.begin_transaction() as tx: for _ in range(2): result = tx.run(*test_run_args) @@ -180,13 +171,13 @@ def test_closes_connection_after_tx_commit(pool, test_run_args): (None, [], ["abc"], ["foo", "bar"], {"a", "b"}, ("1", "two")) ) @mark_sync_test -def test_session_returns_bookmarks_directly(pool, bookmark_values): +def test_session_returns_bookmarks_directly(fake_pool, bookmark_values): if bookmark_values is not None: bookmarks = Bookmarks.from_raw_values(bookmark_values) else: bookmarks = Bookmarks() with Session( - pool, SessionConfig(bookmarks=bookmarks) + fake_pool, SessionConfig(bookmarks=bookmarks) ) as session: ret_bookmarks = (session.last_bookmarks()) assert isinstance(ret_bookmarks, Bookmarks) @@ -202,12 +193,13 @@ def test_session_returns_bookmarks_directly(pool, bookmark_values): (None, [], ["abc"], ["foo", "bar"], ("1", "two")) ) @mark_sync_test -def test_session_last_bookmark_is_deprecated(pool, bookmarks): +def test_session_last_bookmark_is_deprecated(fake_pool, bookmarks): if bookmarks is not None: with pytest.warns(DeprecationWarning): - session = Session(pool, SessionConfig(bookmarks=bookmarks)) + session = Session(fake_pool, + SessionConfig(bookmarks=bookmarks)) else: - session = Session(pool, SessionConfig(bookmarks=bookmarks)) + session = Session(fake_pool, SessionConfig(bookmarks=bookmarks)) with session: with pytest.warns(DeprecationWarning): if bookmarks: @@ -221,9 +213,11 @@ def test_session_last_bookmark_is_deprecated(pool, bookmarks): (("foo",), ("foo", "bar"), (), ["foo", "bar"], {"a", "b"}) ) @mark_sync_test -def test_session_bookmarks_as_iterable_is_deprecated(pool, bookmarks): +def test_session_bookmarks_as_iterable_is_deprecated( + fake_pool, bookmarks +): with pytest.warns(DeprecationWarning): - with Session(pool, SessionConfig( + with Session(fake_pool, SessionConfig( bookmarks=bookmarks )) as session: ret_bookmarks = (session.last_bookmarks()).raw_values @@ -237,19 +231,19 @@ def test_session_bookmarks_as_iterable_is_deprecated(pool, bookmarks): (["I don't", "think so"], TypeError), )) @mark_sync_test -def test_session_run_wrong_types(pool, query, error_type): - with Session(pool, SessionConfig()) as session: +def test_session_run_wrong_types(fake_pool, query, error_type): + with Session(fake_pool, SessionConfig()) as session: with pytest.raises(error_type): session.run(query) @pytest.mark.parametrize("tx_type", ("write_transaction", "read_transaction")) @mark_sync_test -def test_tx_function_argument_type(pool, tx_type): +def test_tx_function_argument_type(fake_pool, tx_type): def work(tx): assert isinstance(tx, ManagedTransaction) - with Session(pool, SessionConfig()) as session: + with Session(fake_pool, SessionConfig()) as session: getattr(session, tx_type)(work) @@ -262,18 +256,20 @@ def work(tx): )) @mark_sync_test -def test_decorated_tx_function_argument_type(pool, tx_type, decorator_kwargs): +def test_decorated_tx_function_argument_type( + fake_pool, tx_type, decorator_kwargs +): @unit_of_work(**decorator_kwargs) def work(tx): assert isinstance(tx, ManagedTransaction) - with Session(pool, SessionConfig()) as session: + with Session(fake_pool, SessionConfig()) as session: getattr(session, tx_type)(work) @mark_sync_test -def test_session_tx_type(pool): - with Session(pool, SessionConfig()) as session: +def test_session_tx_type(fake_pool): + with Session(fake_pool, SessionConfig()) as session: tx = session.begin_transaction() assert isinstance(tx, Transaction) @@ -300,9 +296,9 @@ def test_session_tx_type(pool): @pytest.mark.parametrize("run_type", ("auto", "unmanaged", "managed")) @mark_sync_test def test_session_run_with_parameters( - pool, parameters, run_type, mocker + fake_pool, parameters, run_type, mocker ): - with Session(pool, SessionConfig()) as session: + with Session(fake_pool, SessionConfig()) as session: if run_type == "auto": session.run("RETURN $x", **parameters) elif run_type == "unmanaged": @@ -315,9 +311,133 @@ def work(tx): else: raise ValueError(run_type) - assert len(pool.acquired_connection_mocks) == 1 - connection_mock = pool.acquired_connection_mocks[0] + assert len(fake_pool.acquired_connection_mocks) == 1 + connection_mock = fake_pool.acquired_connection_mocks[0] assert connection_mock.run.called_once() call = connection_mock.run.call_args assert call.args[0] == "RETURN $x" assert call.kwargs["parameters"] == parameters + + +@pytest.mark.parametrize("db", (None, "adb")) +@pytest.mark.parametrize("routing", (True, False)) +# no home db resolution when connected to Neo4j 4.3 or earlier +@pytest.mark.parametrize("home_db_gets_resolved", (True, False)) +@pytest.mark.parametrize("additional_session_bookmarks", + (None, ["session", "bookmarks"])) +@mark_sync_test +def test_with_bookmark_manager( + fake_pool, db, routing, scripted_connection, home_db_gets_resolved, + additional_session_bookmarks, mocker +): + def update_routing_table_side_effect( + database, imp_user, bookmarks, acquisition_timeout=None, + database_callback=None + ): + if home_db_gets_resolved: + database_callback("homedb") + + def bmm_get_bookmarks(database): + return [f"{database}:bm1"] + + def bmm_gat_all_bookmarks(): + return ["all", "bookmarks"] + + scripted_connection.set_script([ + ("run", {"on_success": None, "on_summary": None}), + ("pull", { + "on_success": ({"bookmark": "res:bm1", "has_more": False},), + "on_summary": None, + "on_records": None, + }) + ]) + fake_pool.buffered_connection_mocks.append(scripted_connection) + + bmm = mocker.Mock(spec=BookmarkManager) + # res_cls_mock = mocker.patch("neo4j._async.work.session.AsyncResult", + # autospec=True) + bmm.get_bookmarks.side_effect = bmm_get_bookmarks + bmm.get_all_bookmarks.side_effect = bmm_gat_all_bookmarks + + if routing: + fake_pool.mock_add_spec(Neo4jPool) + fake_pool.update_routing_table.side_effect = \ + update_routing_table_side_effect + else: + fake_pool.mock_add_spec(BoltPool) + + config = SessionConfig() + config.bookmark_manager = bmm + if db is not None: + config.database = db + if additional_session_bookmarks: + config.bookmarks = Bookmarks.from_raw_values( + additional_session_bookmarks + ) + with Session(fake_pool, config) as session: + assert not bmm.method_calls + + session.run("RETURN 1") + + # assert called bmm accordingly + expected_bmm_method_calls = [mocker.call.get_bookmarks("system"), + mocker.call.get_all_bookmarks()] + if routing and db is None: + expected_bmm_method_calls = [ + # extra call for resolving the home database + mocker.call.get_bookmarks("system"), + *expected_bmm_method_calls + ] + assert bmm.method_calls == expected_bmm_method_calls + assert (bmm.get_bookmarks.call_count + == len(expected_bmm_method_calls) - 1) + bmm.get_all_bookmarks.assert_called_once() + bmm.method_calls.clear() + + expected_update_for_db = db + if not db: + if home_db_gets_resolved and routing: + expected_update_for_db = "homedb" + else: + expected_update_for_db = "" + assert [call[0] for call in bmm.method_calls] == ["update_bookmarks"] + assert bmm.method_calls[0].kwargs == {} + assert len(bmm.method_calls[0].args) == 3 + assert bmm.method_calls[0].args[0] == expected_update_for_db + assert (set(bmm.method_calls[0].args[1]) + == {"all", "bookmarks", *(additional_session_bookmarks or [])}) + assert set(bmm.method_calls[0].args[2]) == {"res:bm1"} + + expected_pool_method_calls = ["acquire", "release"] + if routing and db is None: + expected_pool_method_calls = ["update_routing_table", + *expected_pool_method_calls] + assert ([call[0] for call in fake_pool.method_calls] + == expected_pool_method_calls) + assert (set(fake_pool.acquire.call_args.kwargs["bookmarks"]) + == {"system:bm1", *(additional_session_bookmarks or [])}) + if routing and db is None: + assert ( + set(fake_pool.update_routing_table.call_args.kwargs["bookmarks"]) + == {"system:bm1", *(additional_session_bookmarks or [])} + ) + + assert len(fake_pool.acquired_connection_mocks) == 1 + connection_mock = fake_pool.acquired_connection_mocks[0] + assert connection_mock.run.called_once() + connection_run_call_kwargs = connection_mock.run.call_args.kwargs + assert (set(connection_run_call_kwargs["bookmarks"]) + == {"all", "bookmarks", *(additional_session_bookmarks or [])}) + + +@mark_sync_test +def test_with_ignored_bookmark_manager(fake_pool, mocker): + bmm = mocker.Mock(spec=BookmarkManager) + session_config = SessionConfig() + session_config.bookmark_manager = bmm + session_config.ignore_bookmark_manager = True + with Session(fake_pool, session_config) as session: + session.run("RETURN 1") + + bmm.assert_not_called() + assert not bmm.method_calls From 4712348e6be24787b14526e80613dda457659dfc Mon Sep 17 00:00:00 2001 From: Rouven Bauer Date: Thu, 18 Aug 2022 13:35:57 +0200 Subject: [PATCH 14/26] Fix TestKit backend handling of BMM config options --- testkitbackend/_async/requests.py | 2 +- testkitbackend/_sync/requests.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/testkitbackend/_async/requests.py b/testkitbackend/_async/requests.py index d3558ef53..544bbe9f6 100644 --- a/testkitbackend/_async/requests.py +++ b/testkitbackend/_async/requests.py @@ -156,7 +156,7 @@ async def NewDriver(backend, data): if bookmark_manager_config.get("bookmarksSupplierRegistered"): bmm_kwargs["bookmark_supplier"] = bookmark_supplier(backend) if bookmark_manager_config.get("bookmarksConsumerRegistered"): - bmm_kwargs["bookmarks_consumer"] = bookmark_consumer(backend) + bmm_kwargs["bookmark_consumer"] = bookmark_consumer(backend) kwargs["bookmark_manager"] = \ neo4j.AsyncGraphDatabase.bookmark_manager(**bmm_kwargs) diff --git a/testkitbackend/_sync/requests.py b/testkitbackend/_sync/requests.py index 0fed2f19d..44729d335 100644 --- a/testkitbackend/_sync/requests.py +++ b/testkitbackend/_sync/requests.py @@ -156,7 +156,7 @@ def NewDriver(backend, data): if bookmark_manager_config.get("bookmarksSupplierRegistered"): bmm_kwargs["bookmark_supplier"] = bookmark_supplier(backend) if bookmark_manager_config.get("bookmarksConsumerRegistered"): - bmm_kwargs["bookmarks_consumer"] = bookmark_consumer(backend) + bmm_kwargs["bookmark_consumer"] = bookmark_consumer(backend) kwargs["bookmark_manager"] = \ neo4j.GraphDatabase.bookmark_manager(**bmm_kwargs) From 8902770f726c34f5ad17cac43da252f985ebe36a Mon Sep 17 00:00:00 2001 From: Rouven Bauer Date: Thu, 18 Aug 2022 16:48:59 +0200 Subject: [PATCH 15/26] Rename bookmark manager hooks `bookmark_consumer` -> `bookmarks_consumer` `bookmark_supplier` -> `bookmarks_supplier` https://github.com/neo-technology/drivers-adr/pull/31 --- neo4j/_async/bookmark_manager.py | 22 +++++++++++----------- neo4j/_async/driver.py | 14 +++++++------- neo4j/_sync/bookmark_manager.py | 22 +++++++++++----------- neo4j/_sync/driver.py | 14 +++++++------- testkitbackend/_async/backend.py | 4 ++-- testkitbackend/_async/requests.py | 20 ++++++++++---------- testkitbackend/_sync/backend.py | 4 ++-- testkitbackend/_sync/requests.py | 20 ++++++++++---------- tests/unit/async_/test_bookmark_manager.py | 10 +++++----- tests/unit/sync/test_bookmark_manager.py | 10 +++++----- 10 files changed, 70 insertions(+), 70 deletions(-) diff --git a/neo4j/_async/bookmark_manager.py b/neo4j/_async/bookmark_manager.py index d24d381be..95547fe9b 100644 --- a/neo4j/_async/bookmark_manager.py +++ b/neo4j/_async/bookmark_manager.py @@ -47,12 +47,12 @@ def __init__( self, initial_bookmarks: t.Mapping[str, t.Union[Bookmarks, t.Iterable[str]]] = None, - bookmark_supplier: T_BmSupplier = None, - bookmark_consumer: T_BmConsumer = None + bookmarks_supplier: T_BmSupplier = None, + bookmarks_consumer: T_BmConsumer = None ) -> None: super().__init__() - self._bookmark_supplier = bookmark_supplier - self._bookmark_consumer = bookmark_consumer + self._bookmarks_supplier = bookmarks_supplier + self._bookmarks_consumer = bookmarks_consumer if initial_bookmarks is None: initial_bookmarks = {} self._bookmarks = defaultdict( @@ -73,19 +73,19 @@ async def update_bookmarks( curr_bms = self._bookmarks[database] curr_bms.difference_update(prev_bms) curr_bms.update(new_bms) - if self._bookmark_consumer: + if self._bookmarks_consumer: curr_bms_snapshot = Bookmarks.from_raw_values(curr_bms) - if self._bookmark_consumer: + if self._bookmarks_consumer: await AsyncUtil.callback( - self._bookmark_consumer, database, curr_bms_snapshot + self._bookmarks_consumer, database, curr_bms_snapshot ) async def get_bookmarks(self, database: str) -> t.Set[str]: with self._lock: bms = set(self._bookmarks[database]) - if self._bookmark_supplier: + if self._bookmarks_supplier: extra_bms = await AsyncUtil.callback( - self._bookmark_supplier, database + self._bookmarks_supplier, database ) bms.update(extra_bms.raw_values) return bms @@ -95,9 +95,9 @@ async def get_all_bookmarks(self) -> t.Set[str]: with self._lock: for database in self._bookmarks.keys(): bms.update(self._bookmarks[database]) - if self._bookmark_supplier: + if self._bookmarks_supplier: extra_bms = await AsyncUtil.callback( - self._bookmark_supplier, None + self._bookmarks_supplier, None ) bms.update(extra_bms.raw_values) return bms diff --git a/neo4j/_async/driver.py b/neo4j/_async/driver.py index df846b87b..b975c49a3 100644 --- a/neo4j/_async/driver.py +++ b/neo4j/_async/driver.py @@ -220,8 +220,8 @@ def bookmark_manager( cls, initial_bookmarks: t.Mapping[str, t.Union[Bookmarks, t.Iterable[str]]] = None, - bookmark_supplier: _T_BmSupplier = None, - bookmark_consumer: _T_BmConsumer = None + bookmarks_supplier: _T_BmSupplier = None, + bookmarks_consumer: _T_BmConsumer = None ) -> AsyncBookmarkManager: """Create a default :class:`.AsyncBookmarkManager`. @@ -244,17 +244,17 @@ def bookmark_manager( use this to initialize its internal bookmarks per database. If present, this parameter must be a mapping of database names to :class:`.Bookmarks` or an iterable of raw bookmark values (str). - :param bookmark_supplier: + :param bookmarks_supplier: Function which will be called every time the default bookmark manager's method :meth:`.AsyncBookmarkManager.get_bookmarks` or :meth:`.AsyncBookmarkManager.get_all_bookmarks` gets called. The function will be passed the name of the database (``str``) if ``.get_bookmarks`` is called or ``None`` if ``.get_all_bookmarks`` is called. The function must return a :class:`.Bookmarks` object. - The result of ``bookmark_supplier`` will then be concatenated with + The result of ``bookmarks_supplier`` will then be concatenated with the internal set of bookmarks and used to configure the session in creation. - :param bookmark_consumer: + :param bookmarks_consumer: Function which will be called whenever the set of bookmarks handled by the bookmark manager gets updated with the new internal bookmark set. It will receive the name of the database @@ -266,8 +266,8 @@ def bookmark_manager( """ return AsyncNeo4jBookmarkManager( initial_bookmarks=initial_bookmarks, - bookmark_supplier=bookmark_supplier, - bookmark_consumer=bookmark_consumer + bookmarks_supplier=bookmarks_supplier, + bookmarks_consumer=bookmarks_consumer ) @classmethod diff --git a/neo4j/_sync/bookmark_manager.py b/neo4j/_sync/bookmark_manager.py index 9ae230fa6..d2872d7e3 100644 --- a/neo4j/_sync/bookmark_manager.py +++ b/neo4j/_sync/bookmark_manager.py @@ -47,12 +47,12 @@ def __init__( self, initial_bookmarks: t.Mapping[str, t.Union[Bookmarks, t.Iterable[str]]] = None, - bookmark_supplier: T_BmSupplier = None, - bookmark_consumer: T_BmConsumer = None + bookmarks_supplier: T_BmSupplier = None, + bookmarks_consumer: T_BmConsumer = None ) -> None: super().__init__() - self._bookmark_supplier = bookmark_supplier - self._bookmark_consumer = bookmark_consumer + self._bookmarks_supplier = bookmarks_supplier + self._bookmarks_consumer = bookmarks_consumer if initial_bookmarks is None: initial_bookmarks = {} self._bookmarks = defaultdict( @@ -73,19 +73,19 @@ def update_bookmarks( curr_bms = self._bookmarks[database] curr_bms.difference_update(prev_bms) curr_bms.update(new_bms) - if self._bookmark_consumer: + if self._bookmarks_consumer: curr_bms_snapshot = Bookmarks.from_raw_values(curr_bms) - if self._bookmark_consumer: + if self._bookmarks_consumer: Util.callback( - self._bookmark_consumer, database, curr_bms_snapshot + self._bookmarks_consumer, database, curr_bms_snapshot ) def get_bookmarks(self, database: str) -> t.Set[str]: with self._lock: bms = set(self._bookmarks[database]) - if self._bookmark_supplier: + if self._bookmarks_supplier: extra_bms = Util.callback( - self._bookmark_supplier, database + self._bookmarks_supplier, database ) bms.update(extra_bms.raw_values) return bms @@ -95,9 +95,9 @@ def get_all_bookmarks(self) -> t.Set[str]: with self._lock: for database in self._bookmarks.keys(): bms.update(self._bookmarks[database]) - if self._bookmark_supplier: + if self._bookmarks_supplier: extra_bms = Util.callback( - self._bookmark_supplier, None + self._bookmarks_supplier, None ) bms.update(extra_bms.raw_values) return bms diff --git a/neo4j/_sync/driver.py b/neo4j/_sync/driver.py index 442e5a7f6..4b73ebe3a 100644 --- a/neo4j/_sync/driver.py +++ b/neo4j/_sync/driver.py @@ -219,8 +219,8 @@ def bookmark_manager( cls, initial_bookmarks: t.Mapping[str, t.Union[Bookmarks, t.Iterable[str]]] = None, - bookmark_supplier: _T_BmSupplier = None, - bookmark_consumer: _T_BmConsumer = None + bookmarks_supplier: _T_BmSupplier = None, + bookmarks_consumer: _T_BmConsumer = None ) -> BookmarkManager: """Create a default :class:`.BookmarkManager`. @@ -243,17 +243,17 @@ def bookmark_manager( use this to initialize its internal bookmarks per database. If present, this parameter must be a mapping of database names to :class:`.Bookmarks` or an iterable of raw bookmark values (str). - :param bookmark_supplier: + :param bookmarks_supplier: Function which will be called every time the default bookmark manager's method :meth:`.BookmarkManager.get_bookmarks` or :meth:`.BookmarkManager.get_all_bookmarks` gets called. The function will be passed the name of the database (``str``) if ``.get_bookmarks`` is called or ``None`` if ``.get_all_bookmarks`` is called. The function must return a :class:`.Bookmarks` object. - The result of ``bookmark_supplier`` will then be concatenated with + The result of ``bookmarks_supplier`` will then be concatenated with the internal set of bookmarks and used to configure the session in creation. - :param bookmark_consumer: + :param bookmarks_consumer: Function which will be called whenever the set of bookmarks handled by the bookmark manager gets updated with the new internal bookmark set. It will receive the name of the database @@ -265,8 +265,8 @@ def bookmark_manager( """ return Neo4jBookmarkManager( initial_bookmarks=initial_bookmarks, - bookmark_supplier=bookmark_supplier, - bookmark_consumer=bookmark_consumer + bookmarks_supplier=bookmarks_supplier, + bookmarks_consumer=bookmarks_consumer ) @classmethod diff --git a/testkitbackend/_async/backend.py b/testkitbackend/_async/backend.py index 176b8f598..0d95c697a 100644 --- a/testkitbackend/_async/backend.py +++ b/testkitbackend/_async/backend.py @@ -55,8 +55,8 @@ def __init__(self, rd, wr): self.drivers = {} self.custom_resolutions = {} self.dns_resolutions = {} - self.bookmark_consumptions = {} - self.bookmark_supplies = {} + self.bookmarks_consumptions = {} + self.bookmarks_supplies = {} self.sessions = {} self.results = {} self.errors = {} diff --git a/testkitbackend/_async/requests.py b/testkitbackend/_async/requests.py index 544bbe9f6..51f85d6e8 100644 --- a/testkitbackend/_async/requests.py +++ b/testkitbackend/_async/requests.py @@ -154,9 +154,9 @@ async def NewDriver(backend, data): bmm_kwargs["initial_bookmarks"] = \ bookmark_manager_config.get("initialBookmarks") if bookmark_manager_config.get("bookmarksSupplierRegistered"): - bmm_kwargs["bookmark_supplier"] = bookmark_supplier(backend) + bmm_kwargs["bookmarks_supplier"] = bookmarks_supplier(backend) if bookmark_manager_config.get("bookmarksConsumerRegistered"): - bmm_kwargs["bookmark_consumer"] = bookmark_consumer(backend) + bmm_kwargs["bookmarks_consumer"] = bookmarks_consumer(backend) kwargs["bookmark_manager"] = \ neo4j.AsyncGraphDatabase.bookmark_manager(**bmm_kwargs) @@ -257,7 +257,7 @@ async def DomainNameResolutionCompleted(backend, data): backend.dns_resolutions[data["requestId"]] = data["addresses"] -def bookmark_supplier(backend): +def bookmarks_supplier(backend): async def supplier(database): key = backend.next_key() await backend.send_response("BookmarksSupplierRequest", { @@ -267,22 +267,22 @@ async def supplier(database): if not await backend.process_request(): # connection was closed before end of next message return [] - if key not in backend.bookmark_supplies: + if key not in backend.bookmarks_supplies: raise RuntimeError( "Backend did not receive expected " "BookmarksSupplierCompleted message for id %s" % key ) - return backend.bookmark_supplies.pop(key) + return backend.bookmarks_supplies.pop(key) return supplier async def BookmarksSupplierCompleted(backend, data): - backend.bookmark_supplies[data["requestId"]] = \ + backend.bookmarks_supplies[data["requestId"]] = \ neo4j.Bookmarks.from_raw_values(data["bookmarks"]) -def bookmark_consumer(backend): +def bookmarks_consumer(backend): async def consumer(database, bookmarks): key = backend.next_key() await backend.send_response("BookmarksConsumerRequest", { @@ -293,18 +293,18 @@ async def consumer(database, bookmarks): if not await backend.process_request(): # connection was closed before end of next message return [] - if key not in backend.bookmark_consumptions: + if key not in backend.bookmarks_consumptions: raise RuntimeError( "Backend did not receive expected " "BookmarksConsumerCompleted message for id %s" % key ) - del backend.bookmark_consumptions[key] + del backend.bookmarks_consumptions[key] return consumer async def BookmarksConsumerCompleted(backend, data): - backend.bookmark_consumptions[data["requestId"]] = True + backend.bookmarks_consumptions[data["requestId"]] = True async def DriverClose(backend, data): diff --git a/testkitbackend/_sync/backend.py b/testkitbackend/_sync/backend.py index 5d9b5b8f3..7c9108285 100644 --- a/testkitbackend/_sync/backend.py +++ b/testkitbackend/_sync/backend.py @@ -55,8 +55,8 @@ def __init__(self, rd, wr): self.drivers = {} self.custom_resolutions = {} self.dns_resolutions = {} - self.bookmark_consumptions = {} - self.bookmark_supplies = {} + self.bookmarks_consumptions = {} + self.bookmarks_supplies = {} self.sessions = {} self.results = {} self.errors = {} diff --git a/testkitbackend/_sync/requests.py b/testkitbackend/_sync/requests.py index 44729d335..9fb3974f9 100644 --- a/testkitbackend/_sync/requests.py +++ b/testkitbackend/_sync/requests.py @@ -154,9 +154,9 @@ def NewDriver(backend, data): bmm_kwargs["initial_bookmarks"] = \ bookmark_manager_config.get("initialBookmarks") if bookmark_manager_config.get("bookmarksSupplierRegistered"): - bmm_kwargs["bookmark_supplier"] = bookmark_supplier(backend) + bmm_kwargs["bookmarks_supplier"] = bookmarks_supplier(backend) if bookmark_manager_config.get("bookmarksConsumerRegistered"): - bmm_kwargs["bookmark_consumer"] = bookmark_consumer(backend) + bmm_kwargs["bookmarks_consumer"] = bookmarks_consumer(backend) kwargs["bookmark_manager"] = \ neo4j.GraphDatabase.bookmark_manager(**bmm_kwargs) @@ -257,7 +257,7 @@ def DomainNameResolutionCompleted(backend, data): backend.dns_resolutions[data["requestId"]] = data["addresses"] -def bookmark_supplier(backend): +def bookmarks_supplier(backend): def supplier(database): key = backend.next_key() backend.send_response("BookmarksSupplierRequest", { @@ -267,22 +267,22 @@ def supplier(database): if not backend.process_request(): # connection was closed before end of next message return [] - if key not in backend.bookmark_supplies: + if key not in backend.bookmarks_supplies: raise RuntimeError( "Backend did not receive expected " "BookmarksSupplierCompleted message for id %s" % key ) - return backend.bookmark_supplies.pop(key) + return backend.bookmarks_supplies.pop(key) return supplier def BookmarksSupplierCompleted(backend, data): - backend.bookmark_supplies[data["requestId"]] = \ + backend.bookmarks_supplies[data["requestId"]] = \ neo4j.Bookmarks.from_raw_values(data["bookmarks"]) -def bookmark_consumer(backend): +def bookmarks_consumer(backend): def consumer(database, bookmarks): key = backend.next_key() backend.send_response("BookmarksConsumerRequest", { @@ -293,18 +293,18 @@ def consumer(database, bookmarks): if not backend.process_request(): # connection was closed before end of next message return [] - if key not in backend.bookmark_consumptions: + if key not in backend.bookmarks_consumptions: raise RuntimeError( "Backend did not receive expected " "BookmarksConsumerCompleted message for id %s" % key ) - del backend.bookmark_consumptions[key] + del backend.bookmarks_consumptions[key] return consumer def BookmarksConsumerCompleted(backend, data): - backend.bookmark_consumptions[data["requestId"]] = True + backend.bookmarks_consumptions[data["requestId"]] = True def DriverClose(backend, data): diff --git a/tests/unit/async_/test_bookmark_manager.py b/tests/unit/async_/test_bookmark_manager.py index 24294b9e5..248c67401 100644 --- a/tests/unit/async_/test_bookmark_manager.py +++ b/tests/unit/async_/test_bookmark_manager.py @@ -58,7 +58,7 @@ async def test_return_initial_bookmarks_for_the_given_db(db) -> None: @pytest.mark.parametrize("db", ("db1", "db2", "db3")) @pytest.mark.parametrize("supplier_async", (True, False)) @mark_async_test -async def test_return_get_bookmarks_from_bookmark_supplier( +async def test_return_get_bookmarks_from_bookmarks_supplier( db, mocker, supplier_async ) -> None: if supplier_async and not AsyncUtil.is_async_code: @@ -77,7 +77,7 @@ async def test_return_get_bookmarks_from_bookmark_supplier( ) bmm = neo4j.AsyncGraphDatabase.bookmark_manager( initial_bookmarks=initial_bookmarks, - bookmark_supplier=supplier + bookmarks_supplier=supplier ) assert set(await bmm.get_bookmarks(db)) == { @@ -136,7 +136,7 @@ async def test_return_enriched_bookmarks_list_with_supplied_bookmarks( bmm = neo4j.AsyncGraphDatabase.bookmark_manager( initial_bookmarks=(initial_bookmarks if with_initial_bookmarks else None), - bookmark_supplier=supplier + bookmarks_supplier=supplier ) all_bookmarks = await bmm.get_all_bookmarks() @@ -216,7 +216,7 @@ async def test_notify_on_new_bookmarks( bmm = neo4j.AsyncGraphDatabase.bookmark_manager( initial_bookmarks=(initial_bookmarks if with_initial_bookmarks else None), - bookmark_consumer=consumer + bookmarks_consumer=consumer ) bookmarks_old = {"db1:bm1", "db3:bm1"} bookmarks_new = {"db1:bm4"} @@ -257,7 +257,7 @@ async def test_does_not_notify_on_empty_new_bookmark_set( bmm = neo4j.AsyncGraphDatabase.bookmark_manager( initial_bookmarks=(initial_bookmarks if with_initial_bookmarks else None), - bookmark_consumer=consumer + bookmarks_consumer=consumer ) await bmm.update_bookmarks(db, ["db1:bm1"], []) diff --git a/tests/unit/sync/test_bookmark_manager.py b/tests/unit/sync/test_bookmark_manager.py index eab8f19ef..6aa5d280a 100644 --- a/tests/unit/sync/test_bookmark_manager.py +++ b/tests/unit/sync/test_bookmark_manager.py @@ -58,7 +58,7 @@ def test_return_initial_bookmarks_for_the_given_db(db) -> None: @pytest.mark.parametrize("db", ("db1", "db2", "db3")) @pytest.mark.parametrize("supplier_async", (True, False)) @mark_sync_test -def test_return_get_bookmarks_from_bookmark_supplier( +def test_return_get_bookmarks_from_bookmarks_supplier( db, mocker, supplier_async ) -> None: if supplier_async and not Util.is_async_code: @@ -77,7 +77,7 @@ def test_return_get_bookmarks_from_bookmark_supplier( ) bmm = neo4j.GraphDatabase.bookmark_manager( initial_bookmarks=initial_bookmarks, - bookmark_supplier=supplier + bookmarks_supplier=supplier ) assert set(bmm.get_bookmarks(db)) == { @@ -136,7 +136,7 @@ def test_return_enriched_bookmarks_list_with_supplied_bookmarks( bmm = neo4j.GraphDatabase.bookmark_manager( initial_bookmarks=(initial_bookmarks if with_initial_bookmarks else None), - bookmark_supplier=supplier + bookmarks_supplier=supplier ) all_bookmarks = bmm.get_all_bookmarks() @@ -216,7 +216,7 @@ def test_notify_on_new_bookmarks( bmm = neo4j.GraphDatabase.bookmark_manager( initial_bookmarks=(initial_bookmarks if with_initial_bookmarks else None), - bookmark_consumer=consumer + bookmarks_consumer=consumer ) bookmarks_old = {"db1:bm1", "db3:bm1"} bookmarks_new = {"db1:bm4"} @@ -257,7 +257,7 @@ def test_does_not_notify_on_empty_new_bookmark_set( bmm = neo4j.GraphDatabase.bookmark_manager( initial_bookmarks=(initial_bookmarks if with_initial_bookmarks else None), - bookmark_consumer=consumer + bookmarks_consumer=consumer ) bmm.update_bookmarks(db, ["db1:bm1"], []) From 87f4eaf0fa0c6b729ceec12ca49ccc355a76a91a Mon Sep 17 00:00:00 2001 From: Rouven Bauer Date: Fri, 19 Aug 2022 09:57:48 +0200 Subject: [PATCH 16/26] TestKit add Optimization:MinimalBookmarksSet flag --- testkitbackend/test_config.json | 1 + 1 file changed, 1 insertion(+) diff --git a/testkitbackend/test_config.json b/testkitbackend/test_config.json index e3a364158..18710693f 100644 --- a/testkitbackend/test_config.json +++ b/testkitbackend/test_config.json @@ -49,6 +49,7 @@ "Optimization:ConnectionReuse": true, "Optimization:EagerTransactionBegin": true, "Optimization:ImplicitDefaultArguments": true, + "Optimization:MinimalBookmarksSet": true, "Optimization:MinimalResets": true, "Optimization:PullPipelining": true, "Optimization:ResultListFetchAll": "The idiomatic way to cast to list is indistinguishable from iterating over the result.", From 1e0d440df03237351649509f0e404939b8c4ea69 Mon Sep 17 00:00:00 2001 From: Rouven Bauer Date: Fri, 19 Aug 2022 09:58:15 +0200 Subject: [PATCH 17/26] Add performance warning for when enabling the BMM --- docs/source/api.rst | 8 ++++++++ docs/source/async_api.rst | 9 +++++++++ 2 files changed, 17 insertions(+) diff --git a/docs/source/api.rst b/docs/source/api.rst index caa9a03cf..f83a48bc6 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -428,6 +428,14 @@ manger is used to keep all work on the driver causally consistent. See :class:`.BookmarkManager` for more information. +.. warning:: + Enabling the BookmarkManager can have a negative impact on performance since + all queries will wait for the latest changes to be propagated across the + cluster. + + For simpler use-cases, sessions (:class:`.Session`) can be used to group + a series of queries together that will be causally chained automatically. + :Type: :const:`None` or :class:`.BookmarkManager` :Default: :const:`None` diff --git a/docs/source/async_api.rst b/docs/source/async_api.rst index 28c367a8e..6b1638774 100644 --- a/docs/source/async_api.rst +++ b/docs/source/async_api.rst @@ -205,6 +205,15 @@ manger is used to keep all work on the driver causally consistent. See :class:`BookmarkManager` for more information. +.. warning:: + Enabling the BookmarkManager can have a negative impact on performance since + all queries will wait for the latest changes to be propagated across the + cluster. + + For simpler use-cases, sessions (:class:`.AsyncSession`) can be used to + group a series of queries together that will be causally chained + automatically. + :Type: :const:`None`, :class:`BookmarkManager`, or :class:`AsyncBookmarkManager` :Default: ``None`` From 97189f01322c6397a86e9e405ec55f14eabb7dfc Mon Sep 17 00:00:00 2001 From: Florent Biville Date: Mon, 22 Aug 2022 13:54:01 +0200 Subject: [PATCH 18/26] It's manager... not manger, duh! Signed-off-by: Rouven Bauer --- docs/source/api.rst | 2 +- docs/source/async_api.rst | 2 +- neo4j/_async/driver.py | 2 +- neo4j/_sync/driver.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/source/api.rst b/docs/source/api.rst index f83a48bc6..988176926 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -424,7 +424,7 @@ Specify the client agent name. ``bookmark_manager`` -------------------- Specify a bookmark manager for the driver to use. If present, the bookmark -manger is used to keep all work on the driver causally consistent. +manager is used to keep all work on the driver causally consistent. See :class:`.BookmarkManager` for more information. diff --git a/docs/source/async_api.rst b/docs/source/async_api.rst index 6b1638774..f9412d724 100644 --- a/docs/source/async_api.rst +++ b/docs/source/async_api.rst @@ -201,7 +201,7 @@ For example: ``bookmark_manager`` -------------------- Specify a bookmark manager for the driver to use. If present, the bookmark -manger is used to keep all work on the driver causally consistent. +manager is used to keep all work on the driver causally consistent. See :class:`BookmarkManager` for more information. diff --git a/neo4j/_async/driver.py b/neo4j/_async/driver.py index b975c49a3..c46d124bc 100644 --- a/neo4j/_async/driver.py +++ b/neo4j/_async/driver.py @@ -226,7 +226,7 @@ def bookmark_manager( """Create a default :class:`.AsyncBookmarkManager`. Basic usage example to configure the driver with the default - bookmark manger implementation so that all work is automatically + bookmark manager implementation so that all work is automatically causally chained (i.e., all reads can observe all previous writes even in a clustered setup):: diff --git a/neo4j/_sync/driver.py b/neo4j/_sync/driver.py index 4b73ebe3a..b7dfad281 100644 --- a/neo4j/_sync/driver.py +++ b/neo4j/_sync/driver.py @@ -225,7 +225,7 @@ def bookmark_manager( """Create a default :class:`.BookmarkManager`. Basic usage example to configure the driver with the default - bookmark manger implementation so that all work is automatically + bookmark manager implementation so that all work is automatically causally chained (i.e., all reads can observe all previous writes even in a clustered setup):: From 44eef9847ac99ffb1b462027e170bfc07ae52132 Mon Sep 17 00:00:00 2001 From: Rouven Bauer Date: Mon, 22 Aug 2022 14:13:51 +0200 Subject: [PATCH 19/26] Revert unrelated change to tests The change was meant to get rid of a deprecation warning. It'll be moved into it's own PR. --- tests/unit/async_/test_driver.py | 2 +- tests/unit/sync/test_driver.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/async_/test_driver.py b/tests/unit/async_/test_driver.py index d8664d5db..d9c9096ce 100644 --- a/tests/unit/async_/test_driver.py +++ b/tests/unit/async_/test_driver.py @@ -140,7 +140,7 @@ async def test_routing_driver_constructor(protocol, host, port, params, auth_tok ConfigurationError, "The config settings" ), ( - {"ssl_context": ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)}, + {"ssl_context": ssl.SSLContext(ssl.PROTOCOL_TLSv1)}, ConfigurationError, "The config settings" ), ) diff --git a/tests/unit/sync/test_driver.py b/tests/unit/sync/test_driver.py index f90123399..c97333fe7 100644 --- a/tests/unit/sync/test_driver.py +++ b/tests/unit/sync/test_driver.py @@ -139,7 +139,7 @@ def test_routing_driver_constructor(protocol, host, port, params, auth_token): ConfigurationError, "The config settings" ), ( - {"ssl_context": ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)}, + {"ssl_context": ssl.SSLContext(ssl.PROTOCOL_TLSv1)}, ConfigurationError, "The config settings" ), ) From c166e2be1b3c312cf8c7f4cc597ce7be7949e719 Mon Sep 17 00:00:00 2001 From: Florent Biville Date: Mon, 22 Aug 2022 14:31:34 +0200 Subject: [PATCH 20/26] Clean-up * Avoid calling the BMM implementation that's shipped with the driver "default bookmark manger". This could lead users to believe it's enabled by default. * little oversights Signed-off-by: Rouven Bauer --- neo4j/_async/driver.py | 4 ++-- neo4j/_sync/driver.py | 4 ++-- tests/unit/async_/io/test_class_bolt4x4.py | 2 +- tests/unit/async_/test_driver.py | 10 +++------- tests/unit/sync/io/test_class_bolt4x4.py | 2 +- tests/unit/sync/test_driver.py | 10 +++------- 6 files changed, 12 insertions(+), 20 deletions(-) diff --git a/neo4j/_async/driver.py b/neo4j/_async/driver.py index c46d124bc..ed621ade0 100644 --- a/neo4j/_async/driver.py +++ b/neo4j/_async/driver.py @@ -223,7 +223,7 @@ def bookmark_manager( bookmarks_supplier: _T_BmSupplier = None, bookmarks_consumer: _T_BmConsumer = None ) -> AsyncBookmarkManager: - """Create a default :class:`.AsyncBookmarkManager`. + """Create a :class:`.AsyncBookmarkManager` with default implementation. Basic usage example to configure the driver with the default bookmark manager implementation so that all work is automatically @@ -240,7 +240,7 @@ def bookmark_manager( ) :param initial_bookmarks: - The initial set of bookmarks. The default bookmark manager will + The initial set of bookmarks. The returned bookmark manager will use this to initialize its internal bookmarks per database. If present, this parameter must be a mapping of database names to :class:`.Bookmarks` or an iterable of raw bookmark values (str). diff --git a/neo4j/_sync/driver.py b/neo4j/_sync/driver.py index b7dfad281..82a5ca33c 100644 --- a/neo4j/_sync/driver.py +++ b/neo4j/_sync/driver.py @@ -222,7 +222,7 @@ def bookmark_manager( bookmarks_supplier: _T_BmSupplier = None, bookmarks_consumer: _T_BmConsumer = None ) -> BookmarkManager: - """Create a default :class:`.BookmarkManager`. + """Create a :class:`.BookmarkManager` with default implementation. Basic usage example to configure the driver with the default bookmark manager implementation so that all work is automatically @@ -239,7 +239,7 @@ def bookmark_manager( ) :param initial_bookmarks: - The initial set of bookmarks. The default bookmark manager will + The initial set of bookmarks. The returned bookmark manager will use this to initialize its internal bookmarks per database. If present, this parameter must be a mapping of database names to :class:`.Bookmarks` or an iterable of raw bookmark values (str). diff --git a/tests/unit/async_/io/test_class_bolt4x4.py b/tests/unit/async_/io/test_class_bolt4x4.py index 285aa9744..c88b4af12 100644 --- a/tests/unit/async_/io/test_class_bolt4x4.py +++ b/tests/unit/async_/io/test_class_bolt4x4.py @@ -244,7 +244,7 @@ async def test_hint_recv_timeout_seconds( sockets = fake_socket_pair(address, packer_cls=AsyncBolt4x4.PACKER_CLS, unpacker_cls=AsyncBolt4x4.UNPACKER_CLS) - sockets.client.settimeout = mocker.MagicMock() + sockets.client.settimeout = mocker.Mock() await sockets.server.send_message( b"\x70", {"server": "Neo4j/4.3.4", "hints": hints} ) diff --git a/tests/unit/async_/test_driver.py b/tests/unit/async_/test_driver.py index d9c9096ce..335823835 100644 --- a/tests/unit/async_/test_driver.py +++ b/tests/unit/async_/test_driver.py @@ -202,17 +202,13 @@ def test_driver_trust_config_error( async def test_driver_opens_write_session_by_default(uri, fake_pool, mocker): with expect_async_experimental_warning(): driver = AsyncGraphDatabase.driver(uri) - from neo4j import AsyncTransaction - # we set a specific db, because else the driver would try to fetch a RT # to get hold of the actual home database (which won't work in this # unittest) driver._pool = fake_pool async with driver.session(database="foobar") as session: - # acquire_mock = mocker.patch.object(session._pool, "acquire", - # autospec=True) - tx_mock = mocker.patch("neo4j._async.work.session.AsyncTransaction", - autospec=True) + mocker.patch("neo4j._async.work.session.AsyncTransaction", + autospec=True) tx = await session.begin_transaction() fake_pool.acquire.assert_awaited_once_with( access_mode=WRITE_ACCESS, @@ -303,7 +299,7 @@ async def test_get_server_info_parameters_are_experimental( @mark_async_test -async def test_with_default_bookmark_manager(mocker) -> None: +async def test_with_builtin_bookmark_manager(mocker) -> None: bmm = AsyncGraphDatabase.bookmark_manager() # could be one line, but want to make sure the type checker assigns # bmm whatever type AsyncGraphDatabase.bookmark_manager() returns diff --git a/tests/unit/sync/io/test_class_bolt4x4.py b/tests/unit/sync/io/test_class_bolt4x4.py index 564660966..d15bed04f 100644 --- a/tests/unit/sync/io/test_class_bolt4x4.py +++ b/tests/unit/sync/io/test_class_bolt4x4.py @@ -244,7 +244,7 @@ def test_hint_recv_timeout_seconds( sockets = fake_socket_pair(address, packer_cls=Bolt4x4.PACKER_CLS, unpacker_cls=Bolt4x4.UNPACKER_CLS) - sockets.client.settimeout = mocker.MagicMock() + sockets.client.settimeout = mocker.Mock() sockets.server.send_message( b"\x70", {"server": "Neo4j/4.3.4", "hints": hints} ) diff --git a/tests/unit/sync/test_driver.py b/tests/unit/sync/test_driver.py index c97333fe7..9ebd41b86 100644 --- a/tests/unit/sync/test_driver.py +++ b/tests/unit/sync/test_driver.py @@ -201,17 +201,13 @@ def test_driver_trust_config_error( def test_driver_opens_write_session_by_default(uri, fake_pool, mocker): with expect_async_experimental_warning(): driver = GraphDatabase.driver(uri) - from neo4j import Transaction - # we set a specific db, because else the driver would try to fetch a RT # to get hold of the actual home database (which won't work in this # unittest) driver._pool = fake_pool with driver.session(database="foobar") as session: - # acquire_mock = mocker.patch.object(session._pool, "acquire", - # autospec=True) - tx_mock = mocker.patch("neo4j._sync.work.session.Transaction", - autospec=True) + mocker.patch("neo4j._sync.work.session.Transaction", + autospec=True) tx = session.begin_transaction() fake_pool.acquire.assert_called_once_with( access_mode=WRITE_ACCESS, @@ -302,7 +298,7 @@ def test_get_server_info_parameters_are_experimental( @mark_sync_test -def test_with_default_bookmark_manager(mocker) -> None: +def test_with_builtin_bookmark_manager(mocker) -> None: bmm = GraphDatabase.bookmark_manager() # could be one line, but want to make sure the type checker assigns # bmm whatever type AsyncGraphDatabase.bookmark_manager() returns From 3891c4e826e38b5ec98dfcbea5c4b12c3473793c Mon Sep 17 00:00:00 2001 From: Antonio Barcelos Date: Mon, 22 Aug 2022 15:10:03 +0200 Subject: [PATCH 21/26] Clean-up * Adjust BMM interface: bookmark types are now `Collection`s instead of `Iterable`s. This allows for slightly more efficient BMM implementations without creating significant friction in the usage of BMMs. * Docs typo. Signed-off-by: Rouven Bauer --- docs/source/async_api.rst | 2 +- neo4j/_async/bookmark_manager.py | 14 ++++++-------- neo4j/_sync/bookmark_manager.py | 14 ++++++-------- neo4j/api.py | 8 ++++---- 4 files changed, 17 insertions(+), 21 deletions(-) diff --git a/docs/source/async_api.rst b/docs/source/async_api.rst index f9412d724..7807bdc1b 100644 --- a/docs/source/async_api.rst +++ b/docs/source/async_api.rst @@ -154,7 +154,7 @@ Async Driver Configuration (see :ref:`driver-configuration-ref`). The only differences are: * the async driver accepts an async custom resolver function, see :ref:`async-resolver-ref`. -* the async driver accepts accepts either a :class:`neo4j.api.BookmarkManager` +* the async driver accepts either a :class:`neo4j.api.BookmarkManager` object or a :class:`neo4j.api.AsyncBookmarkManager` as bookmark manager. see :ref:`async-bookmark-manager-ref`. diff --git a/neo4j/_async/bookmark_manager.py b/neo4j/_async/bookmark_manager.py index 95547fe9b..41c098cc7 100644 --- a/neo4j/_async/bookmark_manager.py +++ b/neo4j/_async/bookmark_manager.py @@ -62,17 +62,15 @@ def __init__( self._lock = AsyncCooperativeLock() async def update_bookmarks( - self, database: str, previous_bookmarks: t.Iterable[str], - new_bookmarks: t.Iterable[str] + self, database: str, previous_bookmarks: t.Collection[str], + new_bookmarks: t.Collection[str] ) -> None: - new_bms = set(new_bookmarks) - prev_bms = set(previous_bookmarks) + if not new_bookmarks: + return with self._lock: - if not new_bms: - return curr_bms = self._bookmarks[database] - curr_bms.difference_update(prev_bms) - curr_bms.update(new_bms) + curr_bms.difference_update(previous_bookmarks) + curr_bms.update(new_bookmarks) if self._bookmarks_consumer: curr_bms_snapshot = Bookmarks.from_raw_values(curr_bms) if self._bookmarks_consumer: diff --git a/neo4j/_sync/bookmark_manager.py b/neo4j/_sync/bookmark_manager.py index d2872d7e3..85cb13bfe 100644 --- a/neo4j/_sync/bookmark_manager.py +++ b/neo4j/_sync/bookmark_manager.py @@ -62,17 +62,15 @@ def __init__( self._lock = CooperativeLock() def update_bookmarks( - self, database: str, previous_bookmarks: t.Iterable[str], - new_bookmarks: t.Iterable[str] + self, database: str, previous_bookmarks: t.Collection[str], + new_bookmarks: t.Collection[str] ) -> None: - new_bms = set(new_bookmarks) - prev_bms = set(previous_bookmarks) + if not new_bookmarks: + return with self._lock: - if not new_bms: - return curr_bms = self._bookmarks[database] - curr_bms.difference_update(prev_bms) - curr_bms.update(new_bms) + curr_bms.difference_update(previous_bookmarks) + curr_bms.update(new_bookmarks) if self._bookmarks_consumer: curr_bms_snapshot = Bookmarks.from_raw_values(curr_bms) if self._bookmarks_consumer: diff --git a/neo4j/api.py b/neo4j/api.py index f9a1faece..a95c7216e 100644 --- a/neo4j/api.py +++ b/neo4j/api.py @@ -405,8 +405,8 @@ class BookmarkManager(_Protocol, metaclass=abc.ABCMeta): @abc.abstractmethod def update_bookmarks( - self, database: str, previous_bookmarks: t.Iterable[str], - new_bookmarks: t.Iterable[str] + self, database: str, previous_bookmarks: t.Collection[str], + new_bookmarks: t.Collection[str] ) -> None: """Handle bookmark updates. @@ -461,8 +461,8 @@ class AsyncBookmarkManager(_Protocol, metaclass=abc.ABCMeta): @abc.abstractmethod async def update_bookmarks( - self, database: str, previous_bookmarks: t.Iterable[str], - new_bookmarks: t.Iterable[str] + self, database: str, previous_bookmarks: t.Collection[str], + new_bookmarks: t.Collection[str] ) -> None: ... From 517c07d209b96055e0dd683e341dbbf5759ecd96 Mon Sep 17 00:00:00 2001 From: Rouven Bauer Date: Tue, 23 Aug 2022 09:55:10 +0200 Subject: [PATCH 22/26] Fix session.last_bookmarks() leaking bookmarks from BMM Discovered by Antonio --- neo4j/_async/work/workspace.py | 53 ++++++++++++++------------ neo4j/_sync/work/workspace.py | 53 ++++++++++++++------------ tests/unit/async_/work/test_session.py | 36 ++++++++++++++++- tests/unit/sync/work/test_session.py | 36 ++++++++++++++++- 4 files changed, 126 insertions(+), 52 deletions(-) diff --git a/neo4j/_async/work/workspace.py b/neo4j/_async/work/workspace.py index 3c43c0f37..45f9e5dd5 100644 --- a/neo4j/_async/work/workspace.py +++ b/neo4j/_async/work/workspace.py @@ -100,40 +100,45 @@ def _initialize_bookmarks(self, bookmarks): self._initial_bookmarks = self._bookmarks = prepared_bookmarks async def _get_bookmarks(self, database): - if self._bookmark_manager is not None: - # For 4.3- support: the server will not send the resolved home - # database back. To avoid confusion between `None` as in "all - # database" and `None` as in "home database" we re-write the - # home database to `""`, which otherwise is an invalid database - # name. - if database is None: - database = "" - self._bookmarks = tuple({ - *await AsyncUtil.callback( - self._bookmark_manager.get_bookmarks, database - ), - *self._initial_bookmarks - }) - return self._bookmarks + if self._bookmark_manager is None: + return self._bookmarks + + # For 4.3- support: the server will not send the resolved home + # database back. To avoid confusion between `None` as in "all + # database" and `None` as in "home database" we re-write the + # home database to `""`, which otherwise is an invalid database + # name. It will not work properly either way, as the home database + # can change (server config change or client side user change). + if database is None: + database = "" + self._last_from_bookmark_manager = tuple({ + *await AsyncUtil.callback( + self._bookmark_manager.get_bookmarks, database + ), + *self._initial_bookmarks + }) + return self._last_from_bookmark_manager async def _get_all_bookmarks(self): - if self._bookmark_manager is not None: - self._bookmarks = tuple({ - *await AsyncUtil.callback( - self._bookmark_manager.get_all_bookmarks, - ), - *self._initial_bookmarks - }) - return self._bookmarks + if self._bookmark_manager is None: + return self._bookmarks + + self._last_from_bookmark_manager = tuple({ + *await AsyncUtil.callback( + self._bookmark_manager.get_all_bookmarks, + ), + *self._initial_bookmarks + }) + return self._last_from_bookmark_manager async def _update_bookmarks(self, database, new_bookmarks): if not new_bookmarks: return self._initial_bookmarks = () - previous_bookmarks = self._bookmarks self._bookmarks = new_bookmarks if self._bookmark_manager is None: return + previous_bookmarks = self._last_from_bookmark_manager # For 4.3- support: the server will not send the resolved home # database back. To avoid confusion between `None` as in "all # database" and `None` as in "home database" we re-write the home diff --git a/neo4j/_sync/work/workspace.py b/neo4j/_sync/work/workspace.py index 054d4b41c..844bb96c7 100644 --- a/neo4j/_sync/work/workspace.py +++ b/neo4j/_sync/work/workspace.py @@ -100,40 +100,45 @@ def _initialize_bookmarks(self, bookmarks): self._initial_bookmarks = self._bookmarks = prepared_bookmarks def _get_bookmarks(self, database): - if self._bookmark_manager is not None: - # For 4.3- support: the server will not send the resolved home - # database back. To avoid confusion between `None` as in "all - # database" and `None` as in "home database" we re-write the - # home database to `""`, which otherwise is an invalid database - # name. - if database is None: - database = "" - self._bookmarks = tuple({ - *Util.callback( - self._bookmark_manager.get_bookmarks, database - ), - *self._initial_bookmarks - }) - return self._bookmarks + if self._bookmark_manager is None: + return self._bookmarks + + # For 4.3- support: the server will not send the resolved home + # database back. To avoid confusion between `None` as in "all + # database" and `None` as in "home database" we re-write the + # home database to `""`, which otherwise is an invalid database + # name. It will not work properly either way, as the home database + # can change (server config change or client side user change). + if database is None: + database = "" + self._last_from_bookmark_manager = tuple({ + *Util.callback( + self._bookmark_manager.get_bookmarks, database + ), + *self._initial_bookmarks + }) + return self._last_from_bookmark_manager def _get_all_bookmarks(self): - if self._bookmark_manager is not None: - self._bookmarks = tuple({ - *Util.callback( - self._bookmark_manager.get_all_bookmarks, - ), - *self._initial_bookmarks - }) - return self._bookmarks + if self._bookmark_manager is None: + return self._bookmarks + + self._last_from_bookmark_manager = tuple({ + *Util.callback( + self._bookmark_manager.get_all_bookmarks, + ), + *self._initial_bookmarks + }) + return self._last_from_bookmark_manager def _update_bookmarks(self, database, new_bookmarks): if not new_bookmarks: return self._initial_bookmarks = () - previous_bookmarks = self._bookmarks self._bookmarks = new_bookmarks if self._bookmark_manager is None: return + previous_bookmarks = self._last_from_bookmark_manager # For 4.3- support: the server will not send the resolved home # database back. To avoid confusion between `None` as in "all # database" and `None` as in "home database" we re-write the home diff --git a/tests/unit/async_/work/test_session.py b/tests/unit/async_/work/test_session.py index 8723cf4c4..7c5ab8f26 100644 --- a/tests/unit/async_/work/test_session.py +++ b/tests/unit/async_/work/test_session.py @@ -354,8 +354,6 @@ async def bmm_gat_all_bookmarks(): fake_pool.buffered_connection_mocks.append(async_scripted_connection) bmm = mocker.Mock(spec=AsyncBookmarkManager) - # res_cls_mock = mocker.patch("neo4j._async.work.session.AsyncResult", - # autospec=True) bmm.get_bookmarks.side_effect = bmm_get_bookmarks bmm.get_all_bookmarks.side_effect = bmm_gat_all_bookmarks @@ -430,6 +428,40 @@ async def bmm_gat_all_bookmarks(): == {"all", "bookmarks", *(additional_session_bookmarks or [])}) +@pytest.mark.parametrize("routing", (True, False)) +@pytest.mark.parametrize("session_method", ("run", "get_server_info")) +@mark_async_test +async def test_last_bookmarks_do_not_leak_bookmark_managers_bookmarks( + fake_pool, routing, session_method, mocker +): + async def bmm_get_bookmarks(database): + return [f"bmm:{database}"] + + async def bmm_gat_all_bookmarks(): + return ["bmm:all", "bookmarks"] + + fake_pool.mock_add_spec(AsyncNeo4jPool if routing else AsyncBoltPool) + + bmm = mocker.Mock(spec=AsyncBookmarkManager) + bmm.get_bookmarks.side_effect = bmm_get_bookmarks + bmm.get_all_bookmarks.side_effect = bmm_gat_all_bookmarks + + config = SessionConfig() + config.bookmark_manager = bmm + config.bookmarks = Bookmarks.from_raw_values(["session", "bookmarks"]) + async with AsyncSession(fake_pool, config) as session: + if session_method == "run": + await session.run("RETURN 1") + elif session_method == "get_server_info": + await session._get_server_info() + else: + assert False + last_bookmarks = await session.last_bookmarks() + + assert last_bookmarks.raw_values == {"session", "bookmarks"} + assert last_bookmarks.raw_values == {"session", "bookmarks"} + + @mark_async_test async def test_with_ignored_bookmark_manager(fake_pool, mocker): bmm = mocker.Mock(spec=AsyncBookmarkManager) diff --git a/tests/unit/sync/work/test_session.py b/tests/unit/sync/work/test_session.py index e1fc1b173..cfc672151 100644 --- a/tests/unit/sync/work/test_session.py +++ b/tests/unit/sync/work/test_session.py @@ -354,8 +354,6 @@ def bmm_gat_all_bookmarks(): fake_pool.buffered_connection_mocks.append(scripted_connection) bmm = mocker.Mock(spec=BookmarkManager) - # res_cls_mock = mocker.patch("neo4j._async.work.session.AsyncResult", - # autospec=True) bmm.get_bookmarks.side_effect = bmm_get_bookmarks bmm.get_all_bookmarks.side_effect = bmm_gat_all_bookmarks @@ -430,6 +428,40 @@ def bmm_gat_all_bookmarks(): == {"all", "bookmarks", *(additional_session_bookmarks or [])}) +@pytest.mark.parametrize("routing", (True, False)) +@pytest.mark.parametrize("session_method", ("run", "get_server_info")) +@mark_sync_test +def test_last_bookmarks_do_not_leak_bookmark_managers_bookmarks( + fake_pool, routing, session_method, mocker +): + def bmm_get_bookmarks(database): + return [f"bmm:{database}"] + + def bmm_gat_all_bookmarks(): + return ["bmm:all", "bookmarks"] + + fake_pool.mock_add_spec(Neo4jPool if routing else BoltPool) + + bmm = mocker.Mock(spec=BookmarkManager) + bmm.get_bookmarks.side_effect = bmm_get_bookmarks + bmm.get_all_bookmarks.side_effect = bmm_gat_all_bookmarks + + config = SessionConfig() + config.bookmark_manager = bmm + config.bookmarks = Bookmarks.from_raw_values(["session", "bookmarks"]) + with Session(fake_pool, config) as session: + if session_method == "run": + session.run("RETURN 1") + elif session_method == "get_server_info": + session._get_server_info() + else: + assert False + last_bookmarks = session.last_bookmarks() + + assert last_bookmarks.raw_values == {"session", "bookmarks"} + assert last_bookmarks.raw_values == {"session", "bookmarks"} + + @mark_sync_test def test_with_ignored_bookmark_manager(fake_pool, mocker): bmm = mocker.Mock(spec=BookmarkManager) From f9070cf5d6516cc6bfef5f71ab09c29ce2f6e17c Mon Sep 17 00:00:00 2001 From: Rouven Bauer Date: Tue, 23 Aug 2022 15:54:32 +0200 Subject: [PATCH 23/26] Move BMM config to session level + mark experimental --- docs/source/api.rst | 54 +++++++++------- docs/source/async_api.rst | 60 +++++++++--------- neo4j/_async/driver.py | 64 ++++++++++++------- neo4j/_conf.py | 73 +++++++++++++++++++--- neo4j/_meta.py | 31 ++++----- neo4j/_sync/driver.py | 64 ++++++++++++------- tests/unit/async_/test_bookmark_manager.py | 63 ++++++++----------- tests/unit/async_/test_driver.py | 38 +++++------ tests/unit/common/test_conf.py | 14 ++++- tests/unit/sync/test_bookmark_manager.py | 63 ++++++++----------- tests/unit/sync/test_driver.py | 38 +++++------ 11 files changed, 326 insertions(+), 236 deletions(-) diff --git a/docs/source/api.rst b/docs/source/api.rst index 988176926..5b3f7bf73 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -175,7 +175,6 @@ Additional configuration can be provided via the :class:`neo4j.Driver` construct + :ref:`ssl-context-ref` + :ref:`trusted-certificates-ref` + :ref:`user-agent-ref` -+ :ref:`bookmark-manager-ref` .. _session-connection-timeout-ref: @@ -276,6 +275,7 @@ Specify whether TCP keep-alive should be enabled. :Default: ``True`` **This is experimental.** (See :ref:`filter-warnings-ref`) +It might be changed or removed any time even without prior notice. .. _max-connection-lifetime-ref: @@ -419,29 +419,6 @@ Specify the client agent name. :Default: *The Python Driver will generate a user agent name.* -.. _bookmark-manager-ref: - -``bookmark_manager`` --------------------- -Specify a bookmark manager for the driver to use. If present, the bookmark -manager is used to keep all work on the driver causally consistent. - -See :class:`.BookmarkManager` for more information. - -.. warning:: - Enabling the BookmarkManager can have a negative impact on performance since - all queries will wait for the latest changes to be propagated across the - cluster. - - For simpler use-cases, sessions (:class:`.Session`) can be used to group - a series of queries together that will be causally chained automatically. - -:Type: :const:`None` or :class:`.BookmarkManager` -:Default: :const:`None` - -.. versionadded:: 5.0 - - Driver Object Lifetime ====================== @@ -597,6 +574,7 @@ To construct a :class:`neo4j.Session` use the :meth:`neo4j.Driver.session` metho + :ref:`database-ref` + :ref:`default-access-mode-ref` + :ref:`fetch-size-ref` ++ :ref:`bookmark-manager-ref` .. _bookmarks-ref: @@ -725,6 +703,33 @@ The fetch size used for requesting messages from Neo4j. :Default: ``1000`` +.. _bookmark-manager-ref: + +``bookmark_manager`` +-------------------- +Specify a bookmark manager for the session to use. If present, the bookmark +manager is used to keep all work within the session causally consistent with +all work in other sessions using the same bookmark manager. + +See :class:`.BookmarkManager` for more information. + +.. warning:: + Enabling the BookmarkManager can have a negative impact on performance since + all queries will wait for the latest changes to be propagated across the + cluster. + + For simple use-cases, it often suffices that work within a single session + is automatically causally consistent. + +:Type: :const:`None` or :class:`.BookmarkManager` +:Default: :const:`None` + +.. versionadded:: 5.0 + +**This is experimental.** (See :ref:`filter-warnings-ref`) +It might be changed or removed any time even without prior notice. + + *********** @@ -932,6 +937,7 @@ Graph .. automethod:: relationship_type **This is experimental.** (See :ref:`filter-warnings-ref`) +It might be changed or removed any time even without prior notice. ****** diff --git a/docs/source/async_api.rst b/docs/source/async_api.rst index 7807bdc1b..1c583027c 100644 --- a/docs/source/async_api.rst +++ b/docs/source/async_api.rst @@ -151,12 +151,8 @@ Async Driver Configuration ========================== :class:`neo4j.AsyncDriver` is configured exactly like :class:`neo4j.Driver` -(see :ref:`driver-configuration-ref`). The only differences are: -* the async driver accepts an async custom resolver function, -see :ref:`async-resolver-ref`. -* the async driver accepts either a :class:`neo4j.api.BookmarkManager` -object or a :class:`neo4j.api.AsyncBookmarkManager` as bookmark manager. -see :ref:`async-bookmark-manager-ref`. +(see :ref:`driver-configuration-ref`). The only difference is that the async +driver accepts an async custom resolver function: .. _async-resolver-ref: @@ -196,28 +192,6 @@ For example: :Default: ``None`` -.. _async-bookmark-manager-ref: - -``bookmark_manager`` --------------------- -Specify a bookmark manager for the driver to use. If present, the bookmark -manager is used to keep all work on the driver causally consistent. - -See :class:`BookmarkManager` for more information. - -.. warning:: - Enabling the BookmarkManager can have a negative impact on performance since - all queries will wait for the latest changes to be propagated across the - cluster. - - For simpler use-cases, sessions (:class:`.AsyncSession`) can be used to - group a series of queries together that will be causally chained - automatically. - -:Type: :const:`None`, :class:`BookmarkManager`, or :class:`AsyncBookmarkManager` -:Default: ``None`` - - Driver Object Lifetime ====================== @@ -399,7 +373,35 @@ Session Configuration ===================== :class:`neo4j.AsyncSession` is configured exactly like :class:`neo4j.Session` -(see :ref:`session-configuration-ref`). +(see :ref:`session-configuration-ref`). The only difference is the async session +accepts either a :class:`neo4j.api.BookmarkManager` object or a +:class:`neo4j.api.AsyncBookmarkManager` as bookmark manager: + + +.. _async-bookmark-manager-ref: + +``bookmark_manager`` +-------------------- +Specify a bookmark manager for the driver to use. If present, the bookmark +manager is used to keep all work on the driver causally consistent. + +See :class:`BookmarkManager` for more information. + +.. warning:: + Enabling the BookmarkManager can have a negative impact on performance since + all queries will wait for the latest changes to be propagated across the + cluster. + + For simpler use-cases, sessions (:class:`.AsyncSession`) can be used to + group a series of queries together that will be causally chained + automatically. + +:Type: :const:`None`, :class:`BookmarkManager`, or :class:`AsyncBookmarkManager` +:Default: :const:`None` + +**This is experimental.** (See :ref:`filter-warnings-ref`) +It might be changed or removed any time even without prior notice. + **************** diff --git a/neo4j/_async/driver.py b/neo4j/_async/driver.py index ed621ade0..9df3475dc 100644 --- a/neo4j/_async/driver.py +++ b/neo4j/_async/driver.py @@ -100,8 +100,6 @@ def driver( ssl_context: ssl.SSLContext = ..., user_agent: str = ..., keep_alive: bool = ..., - bookmark_manager: t.Union[AsyncBookmarkManager, - BookmarkManager, None] = ..., # undocumented/unsupported options # they may be change or removed any time without prior notice @@ -112,7 +110,9 @@ def driver( retry_delay_jitter_factor: float = ..., database: t.Optional[str] = ..., fetch_size: int = ..., - impersonated_user: t.Optional[str] = ... + impersonated_user: t.Optional[str] = ..., + bookmark_manager: t.Union[AsyncBookmarkManager, + BookmarkManager, None] = ... ) -> AsyncDriver: ... @@ -216,6 +216,10 @@ def driver(cls, uri, *, auth=None, **config) -> AsyncDriver: routing_context=routing_context, **config) @classmethod + @experimental( + "The bookmark manager feature is experimental. " + "It might be changed or removed any time even without prior notice." + ) def bookmark_manager( cls, initial_bookmarks: t.Mapping[str, t.Union[Bookmarks, @@ -225,19 +229,32 @@ def bookmark_manager( ) -> AsyncBookmarkManager: """Create a :class:`.AsyncBookmarkManager` with default implementation. - Basic usage example to configure the driver with the default - bookmark manager implementation so that all work is automatically - causally chained (i.e., all reads can observe all previous writes - even in a clustered setup):: + Basic usage example to configure sessions with the builtin bookmark + manager implementation so that all work is automatically causally + chained (i.e., all reads can observe all previous writes even in a + clustered setup):: import neo4j - driver = neo4j.AsyncGraphDatabase.driver( - uri, auth=..., # ... - bookmark_manager=neo4j.AsyncGraphDatabase.bookmark_manager( - # ... configure the bookmark manager - ) - ) + driver = neo4j.AsyncGraphDatabase.driver(...) + bookmark_manager = neo4j.AsyncBookmarkManager(...) + + async with driver.session( + bookmark_manager=bookmark_manager + ) as session1: + async with driver.session( + bookmark_manager=bookmark_manager + ) as session2: + session1.run("") + # READ_QUERY is guaranteed to see what WRITE_QUERY wrote. + session2.run("") + + This is a very contrived example, and in this particular case, having + both queries in the same session has the exact same effect and might + even be more performant. However, when dealing with sessions spanning + multiple threads, async Tasks, processes, or even hosts, the bookmark + manager can come in handy as sessions are not safe to be used + concurrently. :param initial_bookmarks: The initial set of bookmarks. The returned bookmark manager will @@ -262,6 +279,9 @@ def bookmark_manager( :returns: A default implementation of :class:`AsyncBookmarkManager`. + **This is experimental.** (See :ref:`filter-warnings-ref`) + It might be changed or removed any time even without prior notice. + .. versionadded:: 5.0 """ return AsyncNeo4jBookmarkManager( @@ -409,14 +429,14 @@ def session( bookmarks: t.Union[t.Iterable[str], Bookmarks, None] = ..., ignore_bookmark_manager: bool = ..., default_access_mode: str = ..., + bookmark_manager: t.Union[AsyncBookmarkManager, + BookmarkManager, None] = ..., # undocumented/unsupported options # they may be change or removed any time without prior notice initial_retry_delay: float = ..., retry_delay_multiplier: float = ..., - retry_delay_jitter_factor: float = ..., - bookmark_manager: t.Union[AsyncBookmarkManager, - BookmarkManager, None] = ..., + retry_delay_jitter_factor: float = ... ) -> AsyncSession: ... @@ -453,13 +473,13 @@ async def verify_connectivity( impersonated_user: t.Optional[str] = ..., bookmarks: t.Union[t.Iterable[str], Bookmarks, None] = ..., default_access_mode: str = ..., + bookmark_manager: t.Union[AsyncBookmarkManager, + BookmarkManager, None] = ..., # undocumented/unsupported options initial_retry_delay: float = ..., retry_delay_multiplier: float = ..., - retry_delay_jitter_factor: float = ..., - bookmark_manager: t.Union[AsyncBookmarkManager, - BookmarkManager, None] = ..., + retry_delay_jitter_factor: float = ... ) -> None: ... @@ -517,13 +537,13 @@ async def get_server_info( impersonated_user: t.Optional[str] = ..., bookmarks: t.Union[t.Iterable[str], Bookmarks, None] = ..., default_access_mode: str = ..., + bookmark_manager: t.Union[AsyncBookmarkManager, + BookmarkManager, None] = ..., # undocumented/unsupported options initial_retry_delay: float = ..., retry_delay_multiplier: float = ..., - retry_delay_jitter_factor: float = ..., - bookmark_manager: t.Union[AsyncBookmarkManager, - BookmarkManager, None] = ..., + retry_delay_jitter_factor: float = ... ) -> ServerInfo: ... diff --git a/neo4j/_conf.py b/neo4j/_conf.py index fab708a5f..434670853 100644 --- a/neo4j/_conf.py +++ b/neo4j/_conf.py @@ -16,11 +16,14 @@ # limitations under the License. +import warnings from abc import ABCMeta from collections.abc import Mapping from ._meta import ( deprecation_warn, + experimental_warn, + ExperimentalWarning, get_user_agent, ) from .api import ( @@ -132,28 +135,59 @@ def __init__(self, new, converter=None): self.converter = converter +class DeprecatedOption: + """Used for deprecated config options without alternative.""" + + def __init__(self, value): + self.value = value + + +class ExperimentalOption: + """Used for experimental config options.""" + + def __init__(self, value): + self.value = value + + class ConfigType(ABCMeta): def __new__(mcs, name, bases, attributes): fields = [] deprecated_aliases = {} deprecated_alternatives = {} + deprecated_options = {} + experimental_options = {} for base in bases: if type(base) is mcs: fields += base.keys() deprecated_aliases.update(base._deprecated_aliases()) deprecated_alternatives.update(base._deprecated_alternatives()) + deprecated_options.update(base._deprecated_options()) + experimental_options.update(base._experimental_options()) for k, v in attributes.items(): + if ( + k.startswith("_") + or callable(v) + or isinstance(v, (staticmethod, classmethod)) + ): + continue if isinstance(v, DeprecatedAlias): deprecated_aliases[k] = v.new - elif isinstance(v, DeprecatedAlternative): + continue + if isinstance(v, DeprecatedAlternative): deprecated_alternatives[k] = v.new, v.converter - elif not (k.startswith("_") - or callable(v) - or isinstance(v, (staticmethod, classmethod))): - fields.append(k) + continue + fields.append(k) + if isinstance(v, DeprecatedOption): + deprecated_options[k] = v.value + attributes[k] = v.value + continue + if isinstance(v, ExperimentalOption): + experimental_options[k] = v.value + attributes[k] = v.value + continue def keys(_): return set(fields) @@ -173,6 +207,12 @@ def _deprecated_aliases(_): def _deprecated_alternatives(_): return deprecated_alternatives + def _deprecated_options(_): + return deprecated_options + + def _experimental_options(_): + return experimental_options + attributes.setdefault("keys", classmethod(keys)) attributes.setdefault("_get_new", classmethod(_get_new)) @@ -182,6 +222,10 @@ def _deprecated_alternatives(_): classmethod(_deprecated_aliases)) attributes.setdefault("_deprecated_alternatives", classmethod(_deprecated_alternatives)) + attributes.setdefault("_deprecated_options", + classmethod(_deprecated_options)) + attributes.setdefault("_experimental_options", + classmethod(_experimental_options)) return super(ConfigType, mcs).__new__( mcs, name, bases, {k: v for k, v in attributes.items() @@ -227,6 +271,15 @@ def __update(self, data): def set_attr(k, v): if k in self.keys(): + if k in self._deprecated_options(): + deprecation_warn("The '{}' config key is " + "deprecated.".format(k)) + if k in self._experimental_options(): + experimental_warn( + "The '{}' config key is experimental. " + "It might be changed or removed any time even without " + "prior notice.".format(k) + ) setattr(self, k, v) elif k in self._deprecated_keys(): k0 = self._get_new(k) @@ -253,7 +306,13 @@ def set_attr(k, v): def __init__(self, *args, **kwargs): for arg in args: - self.__update(arg) + if isinstance(arg, Config): + with warnings.catch_warnings(): + for cat in (DeprecationWarning, ExperimentalWarning): + warnings.filterwarnings("ignore", category=cat) + self.__update(arg) + else: + self.__update(arg) self.__update(kwargs) def __repr__(self): @@ -417,7 +476,7 @@ class WorkspaceConfig(Config): # Note that you need appropriate permissions to do so. #: Bookmark Manager - bookmark_manager = None + bookmark_manager = ExperimentalOption(None) # Specify the bookmark manager to be used for sessions by default. diff --git a/neo4j/_meta.py b/neo4j/_meta.py index 382f40507..3e41e3467 100644 --- a/neo4j/_meta.py +++ b/neo4j/_meta.py @@ -17,10 +17,14 @@ import asyncio +import typing as t from functools import wraps from warnings import warn +_FuncT = t.TypeVar("_FuncT", bound=t.Callable) + + # Can be automatically overridden in builds package = "neo4j" version = "5.0.dev0" @@ -39,22 +43,19 @@ def get_user_agent(): return template.format(*fields) -def deprecation_warn(message, stack_level=1): - warn(message, category=DeprecationWarning, stacklevel=stack_level + 1) +def _id(x): + return x -from typing import ( - Callable, - cast, - TypeVar, -) +def copy_signature(_: _FuncT) -> t.Callable[[t.Callable], _FuncT]: + return _id -T = TypeVar("T") -FuncT = TypeVar("FuncT", bound=Callable[..., object]) +def deprecation_warn(message, stack_level=1): + warn(message, category=DeprecationWarning, stacklevel=stack_level + 1) -def deprecated(message: str) -> Callable[[FuncT], FuncT]: +def deprecated(message: str) -> t.Callable[[_FuncT], _FuncT]: """ Decorator for deprecating functions and methods. :: @@ -64,21 +65,21 @@ def foo(x): pass """ - def decorator(f: FuncT) -> FuncT: + def decorator(f): if asyncio.iscoroutinefunction(f): @wraps(f) async def inner(*args, **kwargs): deprecation_warn(message, stack_level=2) return await f(*args, **kwargs) - return cast(FuncT, inner) + return inner else: @wraps(f) def inner(*args, **kwargs): deprecation_warn(message, stack_level=2) return f(*args, **kwargs) - return cast(FuncT, inner) + return inner return decorator @@ -86,7 +87,7 @@ def inner(*args, **kwargs): def deprecated_property(message: str): def decorator(f): return property(deprecated(message)(f)) - return cast(property, decorator) + return t.cast(property, decorator) class ExperimentalWarning(Warning): @@ -98,7 +99,7 @@ def experimental_warn(message, stack_level=1): warn(message, category=ExperimentalWarning, stacklevel=stack_level + 1) -def experimental(message): +def experimental(message) -> t.Callable[[_FuncT], _FuncT]: """ Decorator for tagging experimental functions and methods. :: diff --git a/neo4j/_sync/driver.py b/neo4j/_sync/driver.py index 82a5ca33c..3fee1d204 100644 --- a/neo4j/_sync/driver.py +++ b/neo4j/_sync/driver.py @@ -99,8 +99,6 @@ def driver( ssl_context: ssl.SSLContext = ..., user_agent: str = ..., keep_alive: bool = ..., - bookmark_manager: t.Union[BookmarkManager, - BookmarkManager, None] = ..., # undocumented/unsupported options # they may be change or removed any time without prior notice @@ -111,7 +109,9 @@ def driver( retry_delay_jitter_factor: float = ..., database: t.Optional[str] = ..., fetch_size: int = ..., - impersonated_user: t.Optional[str] = ... + impersonated_user: t.Optional[str] = ..., + bookmark_manager: t.Union[BookmarkManager, + BookmarkManager, None] = ... ) -> Driver: ... @@ -215,6 +215,10 @@ def driver(cls, uri, *, auth=None, **config) -> Driver: routing_context=routing_context, **config) @classmethod + @experimental( + "The bookmark manager feature is experimental. " + "It might be changed or removed any time even without prior notice." + ) def bookmark_manager( cls, initial_bookmarks: t.Mapping[str, t.Union[Bookmarks, @@ -224,19 +228,32 @@ def bookmark_manager( ) -> BookmarkManager: """Create a :class:`.BookmarkManager` with default implementation. - Basic usage example to configure the driver with the default - bookmark manager implementation so that all work is automatically - causally chained (i.e., all reads can observe all previous writes - even in a clustered setup):: + Basic usage example to configure sessions with the builtin bookmark + manager implementation so that all work is automatically causally + chained (i.e., all reads can observe all previous writes even in a + clustered setup):: import neo4j - driver = neo4j.GraphDatabase.driver( - uri, auth=..., # ... - bookmark_manager=neo4j.GraphDatabase.bookmark_manager( - # ... configure the bookmark manager - ) - ) + driver = neo4j.GraphDatabase.driver(...) + bookmark_manager = neo4j.BookmarkManager(...) + + with driver.session( + bookmark_manager=bookmark_manager + ) as session1: + with driver.session( + bookmark_manager=bookmark_manager + ) as session2: + session1.run("") + # READ_QUERY is guaranteed to see what WRITE_QUERY wrote. + session2.run("") + + This is a very contrived example, and in this particular case, having + both queries in the same session has the exact same effect and might + even be more performant. However, when dealing with sessions spanning + multiple threads, Tasks, processes, or even hosts, the bookmark + manager can come in handy as sessions are not safe to be used + concurrently. :param initial_bookmarks: The initial set of bookmarks. The returned bookmark manager will @@ -261,6 +278,9 @@ def bookmark_manager( :returns: A default implementation of :class:`BookmarkManager`. + **This is experimental.** (See :ref:`filter-warnings-ref`) + It might be changed or removed any time even without prior notice. + .. versionadded:: 5.0 """ return Neo4jBookmarkManager( @@ -408,14 +428,14 @@ def session( bookmarks: t.Union[t.Iterable[str], Bookmarks, None] = ..., ignore_bookmark_manager: bool = ..., default_access_mode: str = ..., + bookmark_manager: t.Union[BookmarkManager, + BookmarkManager, None] = ..., # undocumented/unsupported options # they may be change or removed any time without prior notice initial_retry_delay: float = ..., retry_delay_multiplier: float = ..., - retry_delay_jitter_factor: float = ..., - bookmark_manager: t.Union[BookmarkManager, - BookmarkManager, None] = ..., + retry_delay_jitter_factor: float = ... ) -> Session: ... @@ -452,13 +472,13 @@ def verify_connectivity( impersonated_user: t.Optional[str] = ..., bookmarks: t.Union[t.Iterable[str], Bookmarks, None] = ..., default_access_mode: str = ..., + bookmark_manager: t.Union[BookmarkManager, + BookmarkManager, None] = ..., # undocumented/unsupported options initial_retry_delay: float = ..., retry_delay_multiplier: float = ..., - retry_delay_jitter_factor: float = ..., - bookmark_manager: t.Union[BookmarkManager, - BookmarkManager, None] = ..., + retry_delay_jitter_factor: float = ... ) -> None: ... @@ -516,13 +536,13 @@ def get_server_info( impersonated_user: t.Optional[str] = ..., bookmarks: t.Union[t.Iterable[str], Bookmarks, None] = ..., default_access_mode: str = ..., + bookmark_manager: t.Union[BookmarkManager, + BookmarkManager, None] = ..., # undocumented/unsupported options initial_retry_delay: float = ..., retry_delay_multiplier: float = ..., - retry_delay_jitter_factor: float = ..., - bookmark_manager: t.Union[BookmarkManager, - BookmarkManager, None] = ..., + retry_delay_jitter_factor: float = ... ) -> ServerInfo: ... diff --git a/tests/unit/async_/test_bookmark_manager.py b/tests/unit/async_/test_bookmark_manager.py index 248c67401..9532f670c 100644 --- a/tests/unit/async_/test_bookmark_manager.py +++ b/tests/unit/async_/test_bookmark_manager.py @@ -26,15 +26,26 @@ import neo4j from neo4j._async.bookmark_manager import AsyncNeo4jBookmarkManager from neo4j._async_compat.util import AsyncUtil +from neo4j._meta import copy_signature from neo4j.api import Bookmarks from ..._async_compat import mark_async_test +supplier_async_options = (True, False) if AsyncUtil.is_async_code else (False,) +consumer_async_options = supplier_async_options + + +@copy_signature(neo4j.AsyncGraphDatabase.bookmark_manager) +def bookmark_manager(*args, **kwargs): + with pytest.warns(neo4j.ExperimentalWarning, match="bookmark manager"): + return neo4j.AsyncGraphDatabase.bookmark_manager(*args, **kwargs) + + @pytest.mark.parametrize("db", ("foobar", "system")) @mark_async_test async def test_return_empty_if_db_doesnt_exists(db) -> None: - bmm = neo4j.AsyncGraphDatabase.bookmark_manager() + bmm = bookmark_manager() assert set(await bmm.get_bookmarks(db)) == set() @@ -48,22 +59,17 @@ async def test_return_initial_bookmarks_for_the_given_db(db) -> None: "db3": ["db3:bm1", "db3:bm2"], "db4": ["db4:bm4"] } - bmm = neo4j.AsyncGraphDatabase.bookmark_manager( - initial_bookmarks=initial_bookmarks - ) + bmm = bookmark_manager(initial_bookmarks=initial_bookmarks) assert set(await bmm.get_bookmarks(db)) == set(initial_bookmarks[db]) @pytest.mark.parametrize("db", ("db1", "db2", "db3")) -@pytest.mark.parametrize("supplier_async", (True, False)) +@pytest.mark.parametrize("supplier_async", supplier_async_options) @mark_async_test async def test_return_get_bookmarks_from_bookmarks_supplier( db, mocker, supplier_async ) -> None: - if supplier_async and not AsyncUtil.is_async_code: - pytest.skip("Async only test") - extra_bookmarks = ["foo:bm1", "bar:bm2", "foo:bm1"] initial_bookmarks: t.Dict[str, t.List[str]] = { "db1": ["db1:bm1", "db1:bm1"], @@ -75,10 +81,8 @@ async def test_return_get_bookmarks_from_bookmarks_supplier( supplier = mock_cls( return_value=Bookmarks.from_raw_values(extra_bookmarks) ) - bmm = neo4j.AsyncGraphDatabase.bookmark_manager( - initial_bookmarks=initial_bookmarks, - bookmarks_supplier=supplier - ) + bmm = bookmark_manager(initial_bookmarks=initial_bookmarks, + bookmarks_supplier=supplier) assert set(await bmm.get_bookmarks(db)) == { *extra_bookmarks, *initial_bookmarks.get(db, []) @@ -99,7 +103,7 @@ async def test_return_all_bookmarks(with_initial_bookmarks) -> None: "db4": ["db4:bm4"], "db5": ["db3:bm1"] } - bmm = neo4j.AsyncGraphDatabase.bookmark_manager( + bmm = bookmark_manager( initial_bookmarks=initial_bookmarks if with_initial_bookmarks else None ) @@ -114,14 +118,11 @@ async def test_return_all_bookmarks(with_initial_bookmarks) -> None: @pytest.mark.parametrize("with_initial_bookmarks", (True, False)) -@pytest.mark.parametrize("supplier_async", (True, False)) +@pytest.mark.parametrize("supplier_async", supplier_async_options) @mark_async_test async def test_return_enriched_bookmarks_list_with_supplied_bookmarks( with_initial_bookmarks, supplier_async, mocker ) -> None: - if supplier_async and not AsyncUtil.is_async_code: - pytest.skip("Async only test") - initial_bookmarks: t.Dict[str, t.List[str]] = { "db1": ["db1:bm1", "db1:bm1"], "db2": [], @@ -133,7 +134,7 @@ async def test_return_enriched_bookmarks_list_with_supplied_bookmarks( supplier = mock_cls( return_value=Bookmarks.from_raw_values(extra_bookmarks) ) - bmm = neo4j.AsyncGraphDatabase.bookmark_manager( + bmm = bookmark_manager( initial_bookmarks=(initial_bookmarks if with_initial_bookmarks else None), bookmarks_supplier=supplier @@ -161,9 +162,7 @@ async def test_chains_bookmarks_for_existing_db() -> None: "db3": ["db3:bm1", "db3:bm2"], "db4": ["db4:bm4"], } - bmm = neo4j.AsyncGraphDatabase.bookmark_manager( - initial_bookmarks=initial_bookmarks, - ) + bmm = bookmark_manager(initial_bookmarks=initial_bookmarks) await bmm.update_bookmarks("db3", ["db3:bm1"], ["db3:bm3"]) new_bookmarks = await bmm.get_bookmarks("db3") all_bookmarks = await bmm.get_all_bookmarks() @@ -182,9 +181,7 @@ async def test_add_bookmarks_for_a_non_existing_database() -> None: "db3": ["db3:bm1", "db3:bm2"], "db4": ["db4:bm4"], } - bmm = neo4j.AsyncGraphDatabase.bookmark_manager( - initial_bookmarks=initial_bookmarks, - ) + bmm = bookmark_manager(initial_bookmarks=initial_bookmarks) await bmm.update_bookmarks( "db5", ["db3:bm1", "db5:bm1"], ["db3:bm3", "db3:bm5"] ) @@ -198,22 +195,19 @@ async def test_add_bookmarks_for_a_non_existing_database() -> None: @pytest.mark.parametrize("with_initial_bookmarks", (True, False)) -@pytest.mark.parametrize("consumer_async", (True, False)) +@pytest.mark.parametrize("consumer_async", consumer_async_options) @pytest.mark.parametrize("db", ("db1", "db2", "db3")) @mark_async_test async def test_notify_on_new_bookmarks( with_initial_bookmarks, consumer_async, db, mocker ) -> None: - if consumer_async and not AsyncUtil.is_async_code: - pytest.skip("Async only test") - initial_bookmarks: t.Dict[str, t.List[str]] = { "db1": ["db1:bm1", "db1:bm1", "db1:bm2"], "db2": ["db2:bm1"], } mock_cls = mocker.AsyncMock if consumer_async else mocker.Mock consumer = mock_cls() - bmm = neo4j.AsyncGraphDatabase.bookmark_manager( + bmm = bookmark_manager( initial_bookmarks=(initial_bookmarks if with_initial_bookmarks else None), bookmarks_consumer=consumer @@ -239,22 +233,19 @@ async def test_notify_on_new_bookmarks( assert args[1].raw_values == expected_bms -@pytest.mark.parametrize("consumer_async", (True, False)) +@pytest.mark.parametrize("consumer_async", consumer_async_options) @pytest.mark.parametrize("with_initial_bookmarks", (True, False)) @pytest.mark.parametrize("db", ("db1", "db2")) @mark_async_test async def test_does_not_notify_on_empty_new_bookmark_set( with_initial_bookmarks, consumer_async, db, mocker ) -> None: - if consumer_async and not AsyncUtil.is_async_code: - pytest.skip("Async only test") - initial_bookmarks: t.Dict[str, t.List[str]] = { "db1": ["db1:bm1", "db1:bm2"] } mock_cls = mocker.AsyncMock if consumer_async else mocker.Mock consumer = mock_cls() - bmm = neo4j.AsyncGraphDatabase.bookmark_manager( + bmm = bookmark_manager( initial_bookmarks=(initial_bookmarks if with_initial_bookmarks else None), bookmarks_consumer=consumer @@ -273,9 +264,7 @@ async def test_forget_database(dbs) -> None: "db1": ["db1:bm1", "db1:bm1", "db1:bm2"], "db2": ["db2:bm1"], } - bmm = neo4j.AsyncGraphDatabase.bookmark_manager( - initial_bookmarks=initial_bookmarks - ) + bmm = bookmark_manager(initial_bookmarks=initial_bookmarks) for db in dbs: assert (await bmm.get_bookmarks(db) diff --git a/tests/unit/async_/test_driver.py b/tests/unit/async_/test_driver.py index 449a8db95..3246a47f7 100644 --- a/tests/unit/async_/test_driver.py +++ b/tests/unit/async_/test_driver.py @@ -300,17 +300,17 @@ async def test_get_server_info_parameters_are_experimental( @mark_async_test async def test_with_builtin_bookmark_manager(mocker) -> None: - bmm = AsyncGraphDatabase.bookmark_manager() + with pytest.warns(ExperimentalWarning, match="bookmark manager"): + bmm = AsyncGraphDatabase.bookmark_manager() # could be one line, but want to make sure the type checker assigns # bmm whatever type AsyncGraphDatabase.bookmark_manager() returns session_cls_mock = mocker.patch("neo4j._async.driver.AsyncSession", autospec=True) with expect_async_experimental_warning(): - driver = AsyncGraphDatabase.driver( - "bolt://localhost", bookmark_manager=bmm - ) + driver = AsyncGraphDatabase.driver("bolt://localhost") async with driver as driver: - _ = driver.session() + with pytest.warns(ExperimentalWarning, match="bookmark_manager"): + _ = driver.session(bookmark_manager=bmm) session_cls_mock.assert_called_once() assert session_cls_mock.call_args[0][1].bookmark_manager is bmm @@ -339,11 +339,10 @@ async def forget(self, databases: t.Iterable[str]) -> None: session_cls_mock = mocker.patch("neo4j._async.driver.AsyncSession", autospec=True) with expect_async_experimental_warning(): - driver = AsyncGraphDatabase.driver( - "bolt://localhost", bookmark_manager=bmm - ) + driver = AsyncGraphDatabase.driver("bolt://localhost") async with driver as driver: - _ = driver.session() + with pytest.warns(ExperimentalWarning, match="bookmark_manager"): + _ = driver.session(bookmark_manager=bmm) session_cls_mock.assert_called_once() assert session_cls_mock.call_args[0][1].bookmark_manager is bmm @@ -372,11 +371,10 @@ def forget(self, databases: t.Iterable[str]) -> None: session_cls_mock = mocker.patch("neo4j._async.driver.AsyncSession", autospec=True) with expect_async_experimental_warning(): - driver = AsyncGraphDatabase.driver( - "bolt://localhost", bookmark_manager=bmm - ) + driver = AsyncGraphDatabase.driver("bolt://localhost") async with driver as driver: - _ = driver.session() + with pytest.warns(ExperimentalWarning, match="bookmark_manager"): + _ = driver.session(bookmark_manager=bmm) session_cls_mock.assert_called_once() assert session_cls_mock.call_args[0][1].bookmark_manager is bmm @@ -405,11 +403,10 @@ async def forget(self, databases: t.Iterable[str]) -> None: session_cls_mock = mocker.patch("neo4j._async.driver.AsyncSession", autospec=True) with expect_async_experimental_warning(): - driver = AsyncGraphDatabase.driver( - "bolt://localhost", bookmark_manager=bmm - ) + driver = AsyncGraphDatabase.driver("bolt://localhost") async with driver as driver: - _ = driver.session() + with pytest.warns(ExperimentalWarning, match="bookmark_manager"): + _ = driver.session(bookmark_manager=bmm) session_cls_mock.assert_called_once() assert session_cls_mock.call_args[0][1].bookmark_manager is bmm @@ -438,10 +435,9 @@ def forget(self, databases: t.Iterable[str]) -> None: session_cls_mock = mocker.patch("neo4j._async.driver.AsyncSession", autospec=True) with expect_async_experimental_warning(): - driver = AsyncGraphDatabase.driver( - "bolt://localhost", bookmark_manager=bmm - ) + driver = AsyncGraphDatabase.driver("bolt://localhost") async with driver as driver: - _ = driver.session() + with pytest.warns(ExperimentalWarning, match="bookmark_manager"): + _ = driver.session(bookmark_manager=bmm) session_cls_mock.assert_called_once() assert session_cls_mock.call_args[0][1].bookmark_manager is bmm diff --git a/tests/unit/common/test_conf.py b/tests/unit/common/test_conf.py index 408c60433..8c6601ad5 100644 --- a/tests/unit/common/test_conf.py +++ b/tests/unit/common/test_conf.py @@ -19,6 +19,7 @@ import pytest from neo4j import ( + ExperimentalWarning, TrustAll, TrustCustomCAs, TrustSystemCAs, @@ -184,6 +185,14 @@ def test_pool_config_deprecated_and_new_trust_config(value_trust, "trusted_certificates": trusted_certificates}) +@pytest.mark.parametrize("config_cls", (WorkspaceConfig, SessionConfig)) +def test_bookmark_manager_is_experimental(config_cls): + bmm = object() + with pytest.warns(ExperimentalWarning, match="bookmark_manager"): + config = config_cls.consume({"bookmark_manager": bmm}) + assert config.bookmark_manager is bmm + + def test_config_consume_chain(): test_config = {} @@ -192,7 +201,10 @@ def test_config_consume_chain(): test_config.update(test_session_config) - consumed_pool_config, consumed_session_config = Config.consume_chain(test_config, PoolConfig, SessionConfig) + with pytest.warns(ExperimentalWarning, match="bookmark_manager"): + consumed_pool_config, consumed_session_config = Config.consume_chain( + test_config, PoolConfig, SessionConfig + ) assert isinstance(consumed_pool_config, PoolConfig) assert isinstance(consumed_session_config, SessionConfig) diff --git a/tests/unit/sync/test_bookmark_manager.py b/tests/unit/sync/test_bookmark_manager.py index 6aa5d280a..e7bf9f166 100644 --- a/tests/unit/sync/test_bookmark_manager.py +++ b/tests/unit/sync/test_bookmark_manager.py @@ -25,16 +25,27 @@ import neo4j from neo4j._async_compat.util import Util +from neo4j._meta import copy_signature from neo4j._sync.bookmark_manager import Neo4jBookmarkManager from neo4j.api import Bookmarks from ..._async_compat import mark_sync_test +supplier_async_options = (True, False) if Util.is_async_code else (False,) +consumer_async_options = supplier_async_options + + +@copy_signature(neo4j.GraphDatabase.bookmark_manager) +def bookmark_manager(*args, **kwargs): + with pytest.warns(neo4j.ExperimentalWarning, match="bookmark manager"): + return neo4j.GraphDatabase.bookmark_manager(*args, **kwargs) + + @pytest.mark.parametrize("db", ("foobar", "system")) @mark_sync_test def test_return_empty_if_db_doesnt_exists(db) -> None: - bmm = neo4j.GraphDatabase.bookmark_manager() + bmm = bookmark_manager() assert set(bmm.get_bookmarks(db)) == set() @@ -48,22 +59,17 @@ def test_return_initial_bookmarks_for_the_given_db(db) -> None: "db3": ["db3:bm1", "db3:bm2"], "db4": ["db4:bm4"] } - bmm = neo4j.GraphDatabase.bookmark_manager( - initial_bookmarks=initial_bookmarks - ) + bmm = bookmark_manager(initial_bookmarks=initial_bookmarks) assert set(bmm.get_bookmarks(db)) == set(initial_bookmarks[db]) @pytest.mark.parametrize("db", ("db1", "db2", "db3")) -@pytest.mark.parametrize("supplier_async", (True, False)) +@pytest.mark.parametrize("supplier_async", supplier_async_options) @mark_sync_test def test_return_get_bookmarks_from_bookmarks_supplier( db, mocker, supplier_async ) -> None: - if supplier_async and not Util.is_async_code: - pytest.skip("Async only test") - extra_bookmarks = ["foo:bm1", "bar:bm2", "foo:bm1"] initial_bookmarks: t.Dict[str, t.List[str]] = { "db1": ["db1:bm1", "db1:bm1"], @@ -75,10 +81,8 @@ def test_return_get_bookmarks_from_bookmarks_supplier( supplier = mock_cls( return_value=Bookmarks.from_raw_values(extra_bookmarks) ) - bmm = neo4j.GraphDatabase.bookmark_manager( - initial_bookmarks=initial_bookmarks, - bookmarks_supplier=supplier - ) + bmm = bookmark_manager(initial_bookmarks=initial_bookmarks, + bookmarks_supplier=supplier) assert set(bmm.get_bookmarks(db)) == { *extra_bookmarks, *initial_bookmarks.get(db, []) @@ -99,7 +103,7 @@ def test_return_all_bookmarks(with_initial_bookmarks) -> None: "db4": ["db4:bm4"], "db5": ["db3:bm1"] } - bmm = neo4j.GraphDatabase.bookmark_manager( + bmm = bookmark_manager( initial_bookmarks=initial_bookmarks if with_initial_bookmarks else None ) @@ -114,14 +118,11 @@ def test_return_all_bookmarks(with_initial_bookmarks) -> None: @pytest.mark.parametrize("with_initial_bookmarks", (True, False)) -@pytest.mark.parametrize("supplier_async", (True, False)) +@pytest.mark.parametrize("supplier_async", supplier_async_options) @mark_sync_test def test_return_enriched_bookmarks_list_with_supplied_bookmarks( with_initial_bookmarks, supplier_async, mocker ) -> None: - if supplier_async and not Util.is_async_code: - pytest.skip("Async only test") - initial_bookmarks: t.Dict[str, t.List[str]] = { "db1": ["db1:bm1", "db1:bm1"], "db2": [], @@ -133,7 +134,7 @@ def test_return_enriched_bookmarks_list_with_supplied_bookmarks( supplier = mock_cls( return_value=Bookmarks.from_raw_values(extra_bookmarks) ) - bmm = neo4j.GraphDatabase.bookmark_manager( + bmm = bookmark_manager( initial_bookmarks=(initial_bookmarks if with_initial_bookmarks else None), bookmarks_supplier=supplier @@ -161,9 +162,7 @@ def test_chains_bookmarks_for_existing_db() -> None: "db3": ["db3:bm1", "db3:bm2"], "db4": ["db4:bm4"], } - bmm = neo4j.GraphDatabase.bookmark_manager( - initial_bookmarks=initial_bookmarks, - ) + bmm = bookmark_manager(initial_bookmarks=initial_bookmarks) bmm.update_bookmarks("db3", ["db3:bm1"], ["db3:bm3"]) new_bookmarks = bmm.get_bookmarks("db3") all_bookmarks = bmm.get_all_bookmarks() @@ -182,9 +181,7 @@ def test_add_bookmarks_for_a_non_existing_database() -> None: "db3": ["db3:bm1", "db3:bm2"], "db4": ["db4:bm4"], } - bmm = neo4j.GraphDatabase.bookmark_manager( - initial_bookmarks=initial_bookmarks, - ) + bmm = bookmark_manager(initial_bookmarks=initial_bookmarks) bmm.update_bookmarks( "db5", ["db3:bm1", "db5:bm1"], ["db3:bm3", "db3:bm5"] ) @@ -198,22 +195,19 @@ def test_add_bookmarks_for_a_non_existing_database() -> None: @pytest.mark.parametrize("with_initial_bookmarks", (True, False)) -@pytest.mark.parametrize("consumer_async", (True, False)) +@pytest.mark.parametrize("consumer_async", consumer_async_options) @pytest.mark.parametrize("db", ("db1", "db2", "db3")) @mark_sync_test def test_notify_on_new_bookmarks( with_initial_bookmarks, consumer_async, db, mocker ) -> None: - if consumer_async and not Util.is_async_code: - pytest.skip("Async only test") - initial_bookmarks: t.Dict[str, t.List[str]] = { "db1": ["db1:bm1", "db1:bm1", "db1:bm2"], "db2": ["db2:bm1"], } mock_cls = mocker.Mock if consumer_async else mocker.Mock consumer = mock_cls() - bmm = neo4j.GraphDatabase.bookmark_manager( + bmm = bookmark_manager( initial_bookmarks=(initial_bookmarks if with_initial_bookmarks else None), bookmarks_consumer=consumer @@ -239,22 +233,19 @@ def test_notify_on_new_bookmarks( assert args[1].raw_values == expected_bms -@pytest.mark.parametrize("consumer_async", (True, False)) +@pytest.mark.parametrize("consumer_async", consumer_async_options) @pytest.mark.parametrize("with_initial_bookmarks", (True, False)) @pytest.mark.parametrize("db", ("db1", "db2")) @mark_sync_test def test_does_not_notify_on_empty_new_bookmark_set( with_initial_bookmarks, consumer_async, db, mocker ) -> None: - if consumer_async and not Util.is_async_code: - pytest.skip("Async only test") - initial_bookmarks: t.Dict[str, t.List[str]] = { "db1": ["db1:bm1", "db1:bm2"] } mock_cls = mocker.Mock if consumer_async else mocker.Mock consumer = mock_cls() - bmm = neo4j.GraphDatabase.bookmark_manager( + bmm = bookmark_manager( initial_bookmarks=(initial_bookmarks if with_initial_bookmarks else None), bookmarks_consumer=consumer @@ -273,9 +264,7 @@ def test_forget_database(dbs) -> None: "db1": ["db1:bm1", "db1:bm1", "db1:bm2"], "db2": ["db2:bm1"], } - bmm = neo4j.GraphDatabase.bookmark_manager( - initial_bookmarks=initial_bookmarks - ) + bmm = bookmark_manager(initial_bookmarks=initial_bookmarks) for db in dbs: assert (bmm.get_bookmarks(db) diff --git a/tests/unit/sync/test_driver.py b/tests/unit/sync/test_driver.py index 97a9d79ed..acb72618f 100644 --- a/tests/unit/sync/test_driver.py +++ b/tests/unit/sync/test_driver.py @@ -299,17 +299,17 @@ def test_get_server_info_parameters_are_experimental( @mark_sync_test def test_with_builtin_bookmark_manager(mocker) -> None: - bmm = GraphDatabase.bookmark_manager() + with pytest.warns(ExperimentalWarning, match="bookmark manager"): + bmm = GraphDatabase.bookmark_manager() # could be one line, but want to make sure the type checker assigns # bmm whatever type AsyncGraphDatabase.bookmark_manager() returns session_cls_mock = mocker.patch("neo4j._sync.driver.Session", autospec=True) with expect_async_experimental_warning(): - driver = GraphDatabase.driver( - "bolt://localhost", bookmark_manager=bmm - ) + driver = GraphDatabase.driver("bolt://localhost") with driver as driver: - _ = driver.session() + with pytest.warns(ExperimentalWarning, match="bookmark_manager"): + _ = driver.session(bookmark_manager=bmm) session_cls_mock.assert_called_once() assert session_cls_mock.call_args[0][1].bookmark_manager is bmm @@ -338,11 +338,10 @@ def forget(self, databases: t.Iterable[str]) -> None: session_cls_mock = mocker.patch("neo4j._sync.driver.Session", autospec=True) with expect_async_experimental_warning(): - driver = GraphDatabase.driver( - "bolt://localhost", bookmark_manager=bmm - ) + driver = GraphDatabase.driver("bolt://localhost") with driver as driver: - _ = driver.session() + with pytest.warns(ExperimentalWarning, match="bookmark_manager"): + _ = driver.session(bookmark_manager=bmm) session_cls_mock.assert_called_once() assert session_cls_mock.call_args[0][1].bookmark_manager is bmm @@ -371,11 +370,10 @@ def forget(self, databases: t.Iterable[str]) -> None: session_cls_mock = mocker.patch("neo4j._sync.driver.Session", autospec=True) with expect_async_experimental_warning(): - driver = GraphDatabase.driver( - "bolt://localhost", bookmark_manager=bmm - ) + driver = GraphDatabase.driver("bolt://localhost") with driver as driver: - _ = driver.session() + with pytest.warns(ExperimentalWarning, match="bookmark_manager"): + _ = driver.session(bookmark_manager=bmm) session_cls_mock.assert_called_once() assert session_cls_mock.call_args[0][1].bookmark_manager is bmm @@ -404,11 +402,10 @@ def forget(self, databases: t.Iterable[str]) -> None: session_cls_mock = mocker.patch("neo4j._sync.driver.Session", autospec=True) with expect_async_experimental_warning(): - driver = GraphDatabase.driver( - "bolt://localhost", bookmark_manager=bmm - ) + driver = GraphDatabase.driver("bolt://localhost") with driver as driver: - _ = driver.session() + with pytest.warns(ExperimentalWarning, match="bookmark_manager"): + _ = driver.session(bookmark_manager=bmm) session_cls_mock.assert_called_once() assert session_cls_mock.call_args[0][1].bookmark_manager is bmm @@ -437,10 +434,9 @@ def forget(self, databases: t.Iterable[str]) -> None: session_cls_mock = mocker.patch("neo4j._sync.driver.Session", autospec=True) with expect_async_experimental_warning(): - driver = GraphDatabase.driver( - "bolt://localhost", bookmark_manager=bmm - ) + driver = GraphDatabase.driver("bolt://localhost") with driver as driver: - _ = driver.session() + with pytest.warns(ExperimentalWarning, match="bookmark_manager"): + _ = driver.session(bookmark_manager=bmm) session_cls_mock.assert_called_once() assert session_cls_mock.call_args[0][1].bookmark_manager is bmm From bfbf7e5ff6ae0277ed2c2a1501938fae702b15ff Mon Sep 17 00:00:00 2001 From: Rouven Bauer Date: Mon, 29 Aug 2022 13:28:51 +0200 Subject: [PATCH 24/26] TestKit: support session-level BMM config https://github.com/neo4j-drivers/testkit/pull/509 --- testkitbackend/_async/backend.py | 1 + testkitbackend/_async/requests.py | 46 ++++++++++++++++++++----------- testkitbackend/_sync/backend.py | 1 + testkitbackend/_sync/requests.py | 46 ++++++++++++++++++++----------- 4 files changed, 62 insertions(+), 32 deletions(-) diff --git a/testkitbackend/_async/backend.py b/testkitbackend/_async/backend.py index 0d95c697a..851865c6b 100644 --- a/testkitbackend/_async/backend.py +++ b/testkitbackend/_async/backend.py @@ -55,6 +55,7 @@ def __init__(self, rd, wr): self.drivers = {} self.custom_resolutions = {} self.dns_resolutions = {} + self.bookmark_managers = {} self.bookmarks_consumptions = {} self.bookmarks_supplies = {} self.sessions = {} diff --git a/testkitbackend/_async/requests.py b/testkitbackend/_async/requests.py index 51f85d6e8..33426b849 100644 --- a/testkitbackend/_async/requests.py +++ b/testkitbackend/_async/requests.py @@ -146,20 +146,6 @@ async def NewDriver(backend, data): for cert in data["trustedCertificates"]) kwargs["trusted_certificates"] = neo4j.TrustCustomCAs(*cert_paths) data.mark_item_as_read_if_equals("livenessCheckTimeoutMs", None) - bookmark_manager_config = data.get("bookmarkManager", {}) - if bookmark_manager_config: - bmm_kwargs = {} - bookmark_manager_config.mark_item_as_read("initialBookmarks", - recursive=True) - bmm_kwargs["initial_bookmarks"] = \ - bookmark_manager_config.get("initialBookmarks") - if bookmark_manager_config.get("bookmarksSupplierRegistered"): - bmm_kwargs["bookmarks_supplier"] = bookmarks_supplier(backend) - if bookmark_manager_config.get("bookmarksConsumerRegistered"): - bmm_kwargs["bookmarks_consumer"] = bookmarks_consumer(backend) - - kwargs["bookmark_manager"] = \ - neo4j.AsyncGraphDatabase.bookmark_manager(**bmm_kwargs) driver = neo4j.AsyncGraphDatabase.driver( data["uri"], auth=auth, user_agent=data["userAgent"], **kwargs @@ -257,11 +243,34 @@ async def DomainNameResolutionCompleted(backend, data): backend.dns_resolutions[data["requestId"]] = data["addresses"] -def bookmarks_supplier(backend): +async def NewBookmarkManager(backend, data): + bmm_id = backend.next_key() + + bmm_kwargs = {} + data.mark_item_as_read("initialBookmarks", recursive=True) + bmm_kwargs["initial_bookmarks"] = data.get("initialBookmarks") + if data.get("bookmarksSupplierRegistered"): + bmm_kwargs["bookmarks_supplier"] = bookmarks_supplier(backend, bmm_id) + if data.get("bookmarksConsumerRegistered"): + bmm_kwargs["bookmarks_consumer"] = bookmarks_consumer(backend, bmm_id) + + bmm = neo4j.AsyncGraphDatabase.bookmark_manager(**bmm_kwargs) + backend.bookmark_managers[bmm_id] = bmm + await backend.send_response("BookmarkManager", {"id": bmm_id}) + + +async def BookmarkManagerClose(backend, data): + bmm_id = data["id"] + del backend.bookmark_managers[bmm_id] + await backend.send_response("BookmarkManager", {"id": bmm_id}) + + +def bookmarks_supplier(backend, bmm_id): async def supplier(database): key = backend.next_key() await backend.send_response("BookmarksSupplierRequest", { "id": key, + "bookmarkManagerId": bmm_id, "database": database }) if not await backend.process_request(): @@ -282,11 +291,12 @@ async def BookmarksSupplierCompleted(backend, data): neo4j.Bookmarks.from_raw_values(data["bookmarks"]) -def bookmarks_consumer(backend): +def bookmarks_consumer(backend, bmm_id): async def consumer(database, bookmarks): key = backend.next_key() await backend.send_response("BookmarksConsumerRequest", { "id": key, + "bookmarkManagerId": bmm_id, "database": database, "bookmarks": list(bookmarks.raw_values) }) @@ -349,6 +359,10 @@ async def NewSession(backend, data): config["bookmarks"] = neo4j.Bookmarks.from_raw_values( data["bookmarks"] ) + if data.get("bookmarkManagerId") is not None: + config["bookmark_manager"] = backend.bookmark_managers[ + data["bookmarkManagerId"] + ] for (conf_name, data_name) in ( ("fetch_size", "fetchSize"), ("impersonated_user", "impersonatedUser"), diff --git a/testkitbackend/_sync/backend.py b/testkitbackend/_sync/backend.py index 7c9108285..5dae1753d 100644 --- a/testkitbackend/_sync/backend.py +++ b/testkitbackend/_sync/backend.py @@ -55,6 +55,7 @@ def __init__(self, rd, wr): self.drivers = {} self.custom_resolutions = {} self.dns_resolutions = {} + self.bookmark_managers = {} self.bookmarks_consumptions = {} self.bookmarks_supplies = {} self.sessions = {} diff --git a/testkitbackend/_sync/requests.py b/testkitbackend/_sync/requests.py index 9fb3974f9..f86761c73 100644 --- a/testkitbackend/_sync/requests.py +++ b/testkitbackend/_sync/requests.py @@ -146,20 +146,6 @@ def NewDriver(backend, data): for cert in data["trustedCertificates"]) kwargs["trusted_certificates"] = neo4j.TrustCustomCAs(*cert_paths) data.mark_item_as_read_if_equals("livenessCheckTimeoutMs", None) - bookmark_manager_config = data.get("bookmarkManager", {}) - if bookmark_manager_config: - bmm_kwargs = {} - bookmark_manager_config.mark_item_as_read("initialBookmarks", - recursive=True) - bmm_kwargs["initial_bookmarks"] = \ - bookmark_manager_config.get("initialBookmarks") - if bookmark_manager_config.get("bookmarksSupplierRegistered"): - bmm_kwargs["bookmarks_supplier"] = bookmarks_supplier(backend) - if bookmark_manager_config.get("bookmarksConsumerRegistered"): - bmm_kwargs["bookmarks_consumer"] = bookmarks_consumer(backend) - - kwargs["bookmark_manager"] = \ - neo4j.GraphDatabase.bookmark_manager(**bmm_kwargs) driver = neo4j.GraphDatabase.driver( data["uri"], auth=auth, user_agent=data["userAgent"], **kwargs @@ -257,11 +243,34 @@ def DomainNameResolutionCompleted(backend, data): backend.dns_resolutions[data["requestId"]] = data["addresses"] -def bookmarks_supplier(backend): +def NewBookmarkManager(backend, data): + bmm_id = backend.next_key() + + bmm_kwargs = {} + data.mark_item_as_read("initialBookmarks", recursive=True) + bmm_kwargs["initial_bookmarks"] = data.get("initialBookmarks") + if data.get("bookmarksSupplierRegistered"): + bmm_kwargs["bookmarks_supplier"] = bookmarks_supplier(backend, bmm_id) + if data.get("bookmarksConsumerRegistered"): + bmm_kwargs["bookmarks_consumer"] = bookmarks_consumer(backend, bmm_id) + + bmm = neo4j.GraphDatabase.bookmark_manager(**bmm_kwargs) + backend.bookmark_managers[bmm_id] = bmm + backend.send_response("BookmarkManager", {"id": bmm_id}) + + +def BookmarkManagerClose(backend, data): + bmm_id = data["id"] + del backend.bookmark_managers[bmm_id] + backend.send_response("BookmarkManager", {"id": bmm_id}) + + +def bookmarks_supplier(backend, bmm_id): def supplier(database): key = backend.next_key() backend.send_response("BookmarksSupplierRequest", { "id": key, + "bookmarkManagerId": bmm_id, "database": database }) if not backend.process_request(): @@ -282,11 +291,12 @@ def BookmarksSupplierCompleted(backend, data): neo4j.Bookmarks.from_raw_values(data["bookmarks"]) -def bookmarks_consumer(backend): +def bookmarks_consumer(backend, bmm_id): def consumer(database, bookmarks): key = backend.next_key() backend.send_response("BookmarksConsumerRequest", { "id": key, + "bookmarkManagerId": bmm_id, "database": database, "bookmarks": list(bookmarks.raw_values) }) @@ -349,6 +359,10 @@ def NewSession(backend, data): config["bookmarks"] = neo4j.Bookmarks.from_raw_values( data["bookmarks"] ) + if data.get("bookmarkManagerId") is not None: + config["bookmark_manager"] = backend.bookmark_managers[ + data["bookmarkManagerId"] + ] for (conf_name, data_name) in ( ("fetch_size", "fetchSize"), ("impersonated_user", "impersonatedUser"), From 9f2a47618c087bcc8ed07eebf3c69ea4e728b753 Mon Sep 17 00:00:00 2001 From: Rouven Bauer Date: Mon, 29 Aug 2022 16:00:16 +0200 Subject: [PATCH 25/26] Code-style: expand abbreviations --- testkitbackend/_async/requests.py | 30 +++++++++++++++++------------- testkitbackend/_sync/requests.py | 30 +++++++++++++++++------------- 2 files changed, 34 insertions(+), 26 deletions(-) diff --git a/testkitbackend/_async/requests.py b/testkitbackend/_async/requests.py index 33426b849..e7d6fc0af 100644 --- a/testkitbackend/_async/requests.py +++ b/testkitbackend/_async/requests.py @@ -244,33 +244,37 @@ async def DomainNameResolutionCompleted(backend, data): async def NewBookmarkManager(backend, data): - bmm_id = backend.next_key() + bookmark_manager_id = backend.next_key() bmm_kwargs = {} data.mark_item_as_read("initialBookmarks", recursive=True) bmm_kwargs["initial_bookmarks"] = data.get("initialBookmarks") if data.get("bookmarksSupplierRegistered"): - bmm_kwargs["bookmarks_supplier"] = bookmarks_supplier(backend, bmm_id) + bmm_kwargs["bookmarks_supplier"] = bookmarks_supplier( + backend, bookmark_manager_id + ) if data.get("bookmarksConsumerRegistered"): - bmm_kwargs["bookmarks_consumer"] = bookmarks_consumer(backend, bmm_id) + bmm_kwargs["bookmarks_consumer"] = bookmarks_consumer( + backend, bookmark_manager_id + ) - bmm = neo4j.AsyncGraphDatabase.bookmark_manager(**bmm_kwargs) - backend.bookmark_managers[bmm_id] = bmm - await backend.send_response("BookmarkManager", {"id": bmm_id}) + bookmark_manager = neo4j.AsyncGraphDatabase.bookmark_manager(**bmm_kwargs) + backend.bookmark_managers[bookmark_manager_id] = bookmark_manager + await backend.send_response("BookmarkManager", {"id": bookmark_manager_id}) async def BookmarkManagerClose(backend, data): - bmm_id = data["id"] - del backend.bookmark_managers[bmm_id] - await backend.send_response("BookmarkManager", {"id": bmm_id}) + bookmark_manager_id = data["id"] + del backend.bookmark_managers[bookmark_manager_id] + await backend.send_response("BookmarkManager", {"id": bookmark_manager_id}) -def bookmarks_supplier(backend, bmm_id): +def bookmarks_supplier(backend, bookmark_manager_id): async def supplier(database): key = backend.next_key() await backend.send_response("BookmarksSupplierRequest", { "id": key, - "bookmarkManagerId": bmm_id, + "bookmarkManagerId": bookmark_manager_id, "database": database }) if not await backend.process_request(): @@ -291,12 +295,12 @@ async def BookmarksSupplierCompleted(backend, data): neo4j.Bookmarks.from_raw_values(data["bookmarks"]) -def bookmarks_consumer(backend, bmm_id): +def bookmarks_consumer(backend, bookmark_manager_id): async def consumer(database, bookmarks): key = backend.next_key() await backend.send_response("BookmarksConsumerRequest", { "id": key, - "bookmarkManagerId": bmm_id, + "bookmarkManagerId": bookmark_manager_id, "database": database, "bookmarks": list(bookmarks.raw_values) }) diff --git a/testkitbackend/_sync/requests.py b/testkitbackend/_sync/requests.py index f86761c73..93e09ab62 100644 --- a/testkitbackend/_sync/requests.py +++ b/testkitbackend/_sync/requests.py @@ -244,33 +244,37 @@ def DomainNameResolutionCompleted(backend, data): def NewBookmarkManager(backend, data): - bmm_id = backend.next_key() + bookmark_manager_id = backend.next_key() bmm_kwargs = {} data.mark_item_as_read("initialBookmarks", recursive=True) bmm_kwargs["initial_bookmarks"] = data.get("initialBookmarks") if data.get("bookmarksSupplierRegistered"): - bmm_kwargs["bookmarks_supplier"] = bookmarks_supplier(backend, bmm_id) + bmm_kwargs["bookmarks_supplier"] = bookmarks_supplier( + backend, bookmark_manager_id + ) if data.get("bookmarksConsumerRegistered"): - bmm_kwargs["bookmarks_consumer"] = bookmarks_consumer(backend, bmm_id) + bmm_kwargs["bookmarks_consumer"] = bookmarks_consumer( + backend, bookmark_manager_id + ) - bmm = neo4j.GraphDatabase.bookmark_manager(**bmm_kwargs) - backend.bookmark_managers[bmm_id] = bmm - backend.send_response("BookmarkManager", {"id": bmm_id}) + bookmark_manager = neo4j.GraphDatabase.bookmark_manager(**bmm_kwargs) + backend.bookmark_managers[bookmark_manager_id] = bookmark_manager + backend.send_response("BookmarkManager", {"id": bookmark_manager_id}) def BookmarkManagerClose(backend, data): - bmm_id = data["id"] - del backend.bookmark_managers[bmm_id] - backend.send_response("BookmarkManager", {"id": bmm_id}) + bookmark_manager_id = data["id"] + del backend.bookmark_managers[bookmark_manager_id] + backend.send_response("BookmarkManager", {"id": bookmark_manager_id}) -def bookmarks_supplier(backend, bmm_id): +def bookmarks_supplier(backend, bookmark_manager_id): def supplier(database): key = backend.next_key() backend.send_response("BookmarksSupplierRequest", { "id": key, - "bookmarkManagerId": bmm_id, + "bookmarkManagerId": bookmark_manager_id, "database": database }) if not backend.process_request(): @@ -291,12 +295,12 @@ def BookmarksSupplierCompleted(backend, data): neo4j.Bookmarks.from_raw_values(data["bookmarks"]) -def bookmarks_consumer(backend, bmm_id): +def bookmarks_consumer(backend, bookmark_manager_id): def consumer(database, bookmarks): key = backend.next_key() backend.send_response("BookmarksConsumerRequest", { "id": key, - "bookmarkManagerId": bmm_id, + "bookmarkManagerId": bookmark_manager_id, "database": database, "bookmarks": list(bookmarks.raw_values) }) From af1311393d9ba30ac5c569e3ac4e0e270e683478 Mon Sep 17 00:00:00 2001 From: Robsdedude Date: Tue, 30 Aug 2022 11:00:02 +0200 Subject: [PATCH 26/26] Fix spelling Co-authored-by: Florent Biville <445792+fbiville@users.noreply.github.com> --- neo4j/_async/driver.py | 2 +- neo4j/_sync/driver.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/neo4j/_async/driver.py b/neo4j/_async/driver.py index 9df3475dc..64d275ec2 100644 --- a/neo4j/_async/driver.py +++ b/neo4j/_async/driver.py @@ -229,7 +229,7 @@ def bookmark_manager( ) -> AsyncBookmarkManager: """Create a :class:`.AsyncBookmarkManager` with default implementation. - Basic usage example to configure sessions with the builtin bookmark + Basic usage example to configure sessions with the built-in bookmark manager implementation so that all work is automatically causally chained (i.e., all reads can observe all previous writes even in a clustered setup):: diff --git a/neo4j/_sync/driver.py b/neo4j/_sync/driver.py index 3fee1d204..a7a884de8 100644 --- a/neo4j/_sync/driver.py +++ b/neo4j/_sync/driver.py @@ -228,7 +228,7 @@ def bookmark_manager( ) -> BookmarkManager: """Create a :class:`.BookmarkManager` with default implementation. - Basic usage example to configure sessions with the builtin bookmark + Basic usage example to configure sessions with the built-in bookmark manager implementation so that all work is automatically causally chained (i.e., all reads can observe all previous writes even in a clustered setup)::