From 9b8fc88a64a618ce5a46f6a00fdb11ae64aa8260 Mon Sep 17 00:00:00 2001 From: "Michael J. Brim" Date: Fri, 19 Jul 2024 13:35:22 -0400 Subject: [PATCH 01/10] Multi-capability and svc2svc support for services This changeset includes updates to enable each IntersectService instance to provide service for a set of capabilities, rather than just one. From the client perspective, the key difference is that operation names should be prepended with the advertised name of the capability providing the operation. Also included is support for service-to-service requests, as is necessary to enable making external requests to other services/capabilities as part of the implementation of any service methods. An additional thread is used to process these external requests asynchronously from incoming request handlers. --- examples/1_hello_world/hello_client.py | 2 +- examples/1_hello_world/hello_service.py | 3 +- examples/1_hello_world_amqp/hello_client.py | 2 +- examples/1_hello_world_amqp/hello_service.py | 3 +- examples/1_hello_world_events/hello_client.py | 2 +- .../1_hello_world_events/hello_service.py | 3 +- examples/2_counting/counting_client.py | 14 +- examples/2_counting/counting_service.py | 3 +- .../2_counting_events/counting_service.py | 3 +- examples/3_ping_pong_events/ping_service.py | 4 +- examples/3_ping_pong_events/pong_service.py | 4 +- examples/3_ping_pong_events/service_runner.py | 3 +- .../3_ping_pong_events_amqp/ping_service.py | 4 +- .../3_ping_pong_events_amqp/pong_service.py | 4 +- .../3_ping_pong_events_amqp/service_runner.py | 3 +- examples/SUBMISSION_RULES.md | 2 +- .../_internal/function_metadata.py | 4 + .../_internal/messages/userspace.py | 4 +- src/intersect_sdk/_internal/schema.py | 118 +++--- src/intersect_sdk/capability/base.py | 15 + src/intersect_sdk/client.py | 20 +- .../client_callback_definitions.py | 6 +- src/intersect_sdk/schema.py | 13 +- src/intersect_sdk/service.py | 346 ++++++++++++++++-- tests/e2e/test_examples.py | 14 +- tests/fixtures/example_schema.json | 2 +- tests/fixtures/example_schema.py | 1 + .../integration/test_return_type_mismatch.py | 10 +- tests/integration/test_service.py | 62 ++-- tests/unit/test_invalid_schema_runtime.py | 2 +- tests/unit/test_schema_valid.py | 43 ++- 31 files changed, 530 insertions(+), 189 deletions(-) diff --git a/examples/1_hello_world/hello_client.py b/examples/1_hello_world/hello_client.py index 5d52bdb..1c7f589 100644 --- a/examples/1_hello_world/hello_client.py +++ b/examples/1_hello_world/hello_client.py @@ -76,7 +76,7 @@ def simple_client_callback( initial_messages = [ IntersectClientMessageParams( destination='hello-organization.hello-facility.hello-system.hello-subsystem.hello-service', - operation='say_hello_to_name', + operation='HelloExample.say_hello_to_name', payload='hello_client', ) ] diff --git a/examples/1_hello_world/hello_service.py b/examples/1_hello_world/hello_service.py index 80c2c8c..12d93db 100644 --- a/examples/1_hello_world/hello_service.py +++ b/examples/1_hello_world/hello_service.py @@ -79,11 +79,12 @@ def say_hello_to_name(self, name: str) -> str: @intersect_message and @intersect_status, and that these functions are appropriately type-annotated. """ capability = HelloServiceCapabilityImplementation() + capability.capability_name = "HelloExample" """ step three - create service from both the configuration and your own capability """ - service = IntersectService(capability, config) + service = IntersectService([capability], config) """ step four - start lifecycle loop. The only necessary parameter is your service. diff --git a/examples/1_hello_world_amqp/hello_client.py b/examples/1_hello_world_amqp/hello_client.py index be3078e..7ce87f4 100644 --- a/examples/1_hello_world_amqp/hello_client.py +++ b/examples/1_hello_world_amqp/hello_client.py @@ -78,7 +78,7 @@ def simple_client_callback( initial_messages = [ IntersectClientMessageParams( destination='hello-organization.hello-facility.hello-system.hello-subsystem.hello-service', - operation='say_hello_to_name', + operation='HelloExample.say_hello_to_name', payload='hello_client', ) ] diff --git a/examples/1_hello_world_amqp/hello_service.py b/examples/1_hello_world_amqp/hello_service.py index 0593c06..30e1ce8 100644 --- a/examples/1_hello_world_amqp/hello_service.py +++ b/examples/1_hello_world_amqp/hello_service.py @@ -80,11 +80,12 @@ def say_hello_to_name(self, name: str) -> str: @intersect_message and @intersect_status, and that these functions are appropriately type-annotated. """ capability = HelloServiceCapabilityImplementation() + capability.capability_name = "HelloExample" """ step three - create service from both the configuration and your own capability """ - service = IntersectService(capability, config) + service = IntersectService([capability], config) """ step four - start lifecycle loop. The only necessary parameter is your service. diff --git a/examples/1_hello_world_events/hello_client.py b/examples/1_hello_world_events/hello_client.py index eee163c..86c44ff 100644 --- a/examples/1_hello_world_events/hello_client.py +++ b/examples/1_hello_world_events/hello_client.py @@ -97,7 +97,7 @@ def simple_event_callback( initial_messages = [ IntersectClientMessageParams( destination='hello-organization.hello-facility.hello-system.hello-subsystem.hello-service', - operation='say_hello_to_name', + operation='HelloExample.say_hello_to_name', payload='hello_client', ) ] diff --git a/examples/1_hello_world_events/hello_service.py b/examples/1_hello_world_events/hello_service.py index 1eae705..4591cbc 100644 --- a/examples/1_hello_world_events/hello_service.py +++ b/examples/1_hello_world_events/hello_service.py @@ -83,11 +83,12 @@ def say_hello_to_name(self, name: str) -> str: @intersect_message and @intersect_status, and that these functions are appropriately type-annotated. """ capability = HelloServiceCapabilityImplementation() + capability.capability_name = "HelloExample" """ step three - create service from both the configuration and your own capability """ - service = IntersectService(capability, config) + service = IntersectService([capability], config) """ step four - start lifecycle loop. The only necessary parameter is your service. diff --git a/examples/2_counting/counting_client.py b/examples/2_counting/counting_client.py index 47fb484..0bf758b 100644 --- a/examples/2_counting/counting_client.py +++ b/examples/2_counting/counting_client.py @@ -40,7 +40,7 @@ def __init__(self) -> None: ( IntersectClientMessageParams( destination='counting-organization.counting-facility.counting-system.counting-subsystem.counting-service', - operation='stop_count', + operation='CountingExample.stop_count', payload=None, ), 5.0, @@ -49,7 +49,7 @@ def __init__(self) -> None: ( IntersectClientMessageParams( destination='counting-organization.counting-facility.counting-system.counting-subsystem.counting-service', - operation='start_count', + operation='CountingExample.start_count', payload=None, ), 1.0, @@ -58,7 +58,7 @@ def __init__(self) -> None: ( IntersectClientMessageParams( destination='counting-organization.counting-facility.counting-system.counting-subsystem.counting-service', - operation='reset_count', + operation='CountingExample.reset_count', payload=True, ), 3.0, @@ -67,7 +67,7 @@ def __init__(self) -> None: ( IntersectClientMessageParams( destination='counting-organization.counting-facility.counting-system.counting-subsystem.counting-service', - operation='reset_count', + operation='CountingExample.reset_count', payload=False, ), 5.0, @@ -76,7 +76,7 @@ def __init__(self) -> None: ( IntersectClientMessageParams( destination='counting-organization.counting-facility.counting-system.counting-subsystem.counting-service', - operation='start_count', + operation='CountingExample.start_count', payload=None, ), 3.0, @@ -85,7 +85,7 @@ def __init__(self) -> None: ( IntersectClientMessageParams( destination='counting-organization.counting-facility.counting-system.counting-subsystem.counting-service', - operation='stop_count', + operation='CountingExample.stop_count', payload=None, ), 3.0, @@ -160,7 +160,7 @@ def client_callback( initial_messages = [ IntersectClientMessageParams( destination='counting-organization.counting-facility.counting-system.counting-subsystem.counting-service', - operation='start_count', + operation='CountingExample.start_count', payload=None, ) ] diff --git a/examples/2_counting/counting_service.py b/examples/2_counting/counting_service.py index 767817c..1ba09c0 100644 --- a/examples/2_counting/counting_service.py +++ b/examples/2_counting/counting_service.py @@ -191,7 +191,8 @@ def _run_count(self) -> None: **from_config_file, ) capability = CountingServiceCapabilityImplementation() - service = IntersectService(capability, config) + capability.capability_name = "CountingExample" + service = IntersectService([capability], config) logger.info('Starting counting_service, use Ctrl+C to exit.') default_intersect_lifecycle_loop( service, diff --git a/examples/2_counting_events/counting_service.py b/examples/2_counting_events/counting_service.py index 13a70d1..d86933d 100644 --- a/examples/2_counting_events/counting_service.py +++ b/examples/2_counting_events/counting_service.py @@ -83,7 +83,8 @@ def increment_counter_function(self) -> None: **from_config_file, ) capability = CountingServiceCapabilityImplementation() - service = IntersectService(capability, config) + capability.capability_name = "CountingExample" + service = IntersectService([capability], config) logger.info('Starting counting_service, use Ctrl+C to exit.') """ diff --git a/examples/3_ping_pong_events/ping_service.py b/examples/3_ping_pong_events/ping_service.py index b117e35..e291024 100644 --- a/examples/3_ping_pong_events/ping_service.py +++ b/examples/3_ping_pong_events/ping_service.py @@ -11,7 +11,7 @@ logging.basicConfig(level=logging.INFO) -class PingCapabiilityImplementation(P_ngBaseCapabilityImplementation): +class PingCapabilityImplementation(P_ngBaseCapabilityImplementation): """Basic capability definition, very similar to the other capability except for the type of event it emits.""" def after_service_startup(self) -> None: @@ -32,4 +32,4 @@ def ping_event(self) -> None: if __name__ == '__main__': - run_service(PingCapabiilityImplementation()) + run_service(PingCapabilityImplementation()) diff --git a/examples/3_ping_pong_events/pong_service.py b/examples/3_ping_pong_events/pong_service.py index f8a13fd..b4f9cb5 100644 --- a/examples/3_ping_pong_events/pong_service.py +++ b/examples/3_ping_pong_events/pong_service.py @@ -11,7 +11,7 @@ logging.basicConfig(level=logging.INFO) -class PongCapabiilityImplementation(P_ngBaseCapabilityImplementation): +class PongCapabilityImplementation(P_ngBaseCapabilityImplementation): """Basic capability definition, very similar to the other capability except for the type of event it emits.""" def after_service_startup(self) -> None: @@ -32,4 +32,4 @@ def pong_event(self) -> None: if __name__ == '__main__': - run_service(PongCapabiilityImplementation()) + run_service(PongCapabilityImplementation()) diff --git a/examples/3_ping_pong_events/service_runner.py b/examples/3_ping_pong_events/service_runner.py index 14d3e81..8bbb93b 100644 --- a/examples/3_ping_pong_events/service_runner.py +++ b/examples/3_ping_pong_events/service_runner.py @@ -58,7 +58,8 @@ def run_service(capability: P_ngBaseCapabilityImplementation) -> None: status_interval=30.0, **from_config_file, ) - service = IntersectService(capability, config) + capability.capability_name = service_name + service = IntersectService([capability], config) logger.info('Starting %s_service, use Ctrl+C to exit.', service_name) """ diff --git a/examples/3_ping_pong_events_amqp/ping_service.py b/examples/3_ping_pong_events_amqp/ping_service.py index d64f8ec..68298b7 100644 --- a/examples/3_ping_pong_events_amqp/ping_service.py +++ b/examples/3_ping_pong_events_amqp/ping_service.py @@ -12,7 +12,7 @@ logging.getLogger('pika').setLevel(logging.WARNING) -class PingCapabiilityImplementation(P_ngBaseCapabilityImplementation): +class PingCapabilityImplementation(P_ngBaseCapabilityImplementation): """Basic capability definition, very similar to the other capability except for the type of event it emits.""" def after_service_startup(self) -> None: @@ -33,4 +33,4 @@ def ping_event(self) -> None: if __name__ == '__main__': - run_service(PingCapabiilityImplementation()) + run_service(PingCapabilityImplementation()) diff --git a/examples/3_ping_pong_events_amqp/pong_service.py b/examples/3_ping_pong_events_amqp/pong_service.py index ac9c15a..ae48410 100644 --- a/examples/3_ping_pong_events_amqp/pong_service.py +++ b/examples/3_ping_pong_events_amqp/pong_service.py @@ -12,7 +12,7 @@ logging.getLogger('pika').setLevel(logging.WARNING) -class PongCapabiilityImplementation(P_ngBaseCapabilityImplementation): +class PongCapabilityImplementation(P_ngBaseCapabilityImplementation): """Basic capability definition, very similar to the other capability except for the type of event it emits.""" def after_service_startup(self) -> None: @@ -33,4 +33,4 @@ def pong_event(self) -> None: if __name__ == '__main__': - run_service(PongCapabiilityImplementation()) + run_service(PongCapabilityImplementation()) diff --git a/examples/3_ping_pong_events_amqp/service_runner.py b/examples/3_ping_pong_events_amqp/service_runner.py index bb71d13..05dbb9f 100644 --- a/examples/3_ping_pong_events_amqp/service_runner.py +++ b/examples/3_ping_pong_events_amqp/service_runner.py @@ -58,7 +58,8 @@ def run_service(capability: P_ngBaseCapabilityImplementation) -> None: status_interval=30.0, **from_config_file, ) - service = IntersectService(capability, config) + capability.capability_name = service_name + service = IntersectService([capability], config) logger.info('Starting %s_service, use Ctrl+C to exit.', service_name) """ diff --git a/examples/SUBMISSION_RULES.md b/examples/SUBMISSION_RULES.md index 389b2c9..7ad282b 100644 --- a/examples/SUBMISSION_RULES.md +++ b/examples/SUBMISSION_RULES.md @@ -77,7 +77,7 @@ import json if __name__ == '__main__': # everything before service creation omitted - service = IntersectService(capability, config) + service = IntersectService([capability], config) print(json.dumps(service._schema, indent=2)) ``` diff --git a/src/intersect_sdk/_internal/function_metadata.py b/src/intersect_sdk/_internal/function_metadata.py index ed49c27..43d6399 100644 --- a/src/intersect_sdk/_internal/function_metadata.py +++ b/src/intersect_sdk/_internal/function_metadata.py @@ -12,6 +12,10 @@ class FunctionMetadata(NamedTuple): NOTE: both this class and all properties in it should remain immutable after creation """ + capability: type + """ + The type of the class that implements the target method. + """ method: Callable[[Any], Any] """ The raw method of the function. The function itself is useless and should not be called, diff --git a/src/intersect_sdk/_internal/messages/userspace.py b/src/intersect_sdk/_internal/messages/userspace.py index e87f6a6..dd06e25 100644 --- a/src/intersect_sdk/_internal/messages/userspace.py +++ b/src/intersect_sdk/_internal/messages/userspace.py @@ -141,11 +141,13 @@ def create_userspace_message( content_type: IntersectMimeType, data_handler: IntersectDataHandler, payload: Any, + message_id: uuid.UUID = None, has_error: bool = False, ) -> UserspaceMessage: """Payloads depend on the data_handler and has_error.""" + msg_id = message_id if message_id else uuid.uuid4() return UserspaceMessage( - messageId=uuid.uuid4(), + messageId=msg_id, operationId=operation_id, contentType=content_type, payload=payload, diff --git a/src/intersect_sdk/_internal/schema.py b/src/intersect_sdk/_internal/schema.py index 63369c4..7abb6c4 100644 --- a/src/intersect_sdk/_internal/schema.py +++ b/src/intersect_sdk/_internal/schema.py @@ -8,6 +8,7 @@ TYPE_CHECKING, Any, Callable, + List, Mapping, NamedTuple, get_origin, @@ -59,7 +60,7 @@ class _FunctionAnalysisResult(NamedTuple): """private class generated from static analysis of function.""" - capability_name: str + class_name: str method_name: str method: Callable[[Any], Any] """raw method is for inspecting attributes""" @@ -218,18 +219,18 @@ def _status_fn_schema( - The status function's schema - The TypeAdapter to use for serializing outgoing responses """ - capability_name, status_fn_name, status_fn, min_params = status_info + class_name, status_fn_name, status_fn, min_params = status_info status_signature = inspect.signature(status_fn) method_params = tuple(status_signature.parameters.values()) if len(method_params) != min_params or any( p.kind not in (p.POSITIONAL_OR_KEYWORD, p.POSITIONAL_ONLY) for p in method_params ): die( - f"On capability '{capability_name}', capability status function '{status_fn_name}' should have no parameters other than 'self' (unless a staticmethod), and should not use keyword or variable length arguments (i.e. '*', *args, **kwargs)." + f"On capability '{class_name}', capability status function '{status_fn_name}' should have no parameters other than 'self' (unless a staticmethod), and should not use keyword or variable length arguments (i.e. '*', *args, **kwargs)." ) if status_signature.return_annotation is inspect.Signature.empty: die( - f"On capability '{capability_name}', capability status function '{status_fn_name}' should have a valid return annotation." + f"On capability '{class_name}', capability status function '{status_fn_name}' should have a valid return annotation." ) try: status_adapter = TypeAdapter(status_signature.return_annotation) @@ -246,12 +247,12 @@ def _status_fn_schema( ) except PydanticUserError as e: die( - f"On capability '{capability_name}', return annotation '{status_signature.return_annotation}' on function '{status_fn_name}' is invalid.\n{e}" + f"On capability '{class_name}', return annotation '{status_signature.return_annotation}' on function '{status_fn_name}' is invalid.\n{e}" ) def _add_events( - capability_name: str, + class_name: str, function_name: str, schemas: dict[str, Any], event_schemas: dict[str, Any], @@ -272,13 +273,13 @@ def _add_events( for d in differences_from_cache ) die( - f"On capability '{capability_name}', event key '{event_key}' on function '{function_name}' was previously defined differently. \n{diff_str}\n" + f"On capability '{class_name}', event key '{event_key}' on function '{function_name}' was previously defined differently. \n{diff_str}\n" ) metadata_value.operations.add(function_name) else: if event_definition.data_handler in excluded_data_handlers: die( - f"On capability '{capability_name}', function '{function_name}' should not set data_handler as {event_definition.data_handler} unless an instance is configured in IntersectConfig.data_stores ." + f"On capability '{class_name}', function '{function_name}' should not set data_handler as {event_definition.data_handler} unless an instance is configured in IntersectConfig.data_stores ." ) try: event_adapter: TypeAdapter[Any] = TypeAdapter(event_definition.event_type) @@ -297,12 +298,12 @@ def _add_events( ) except PydanticUserError as e: die( - f"On capability '{capability_name}', event key '{event_key}' on function '{function_name}' has an invalid value in the events mapping.\n{e}" + f"On capability '{class_name}', event key '{event_key}' on function '{function_name}' has an invalid value in the events mapping.\n{e}" ) def _introspection_baseline( - capability: type[IntersectBaseCapabilityImplementation], + capability: IntersectBaseCapabilityImplementation, excluded_data_handlers: set[IntersectDataHandler], ) -> tuple[ dict[Any, Any], # $defs for schemas (common) @@ -332,10 +333,13 @@ def _introspection_baseline( function_map = {} event_metadatas: dict[str, EventMetadata] = {} - status_func, response_funcs, event_funcs = _get_functions(capability) + cap_name = capability.capability_name + status_func, response_funcs, event_funcs = _get_functions(type(capability)) # parse functions - for capability_name, name, method, min_params in response_funcs: + for class_name, name, method, min_params in response_funcs: + public_name = f'{cap_name}.{name}' + # TODO - I'm placing this here for now because we'll eventually want to capture data plane and broker configs in the schema. # (It's possible we may want to separate the backing service schema from the application logic, but it's unlikely.) # At the moment, we're just validating that users can support their response_data_handler property. @@ -343,7 +347,7 @@ def _introspection_baseline( data_handler = getattr(method, RESPONSE_DATA) if data_handler in excluded_data_handlers: die( - f"On capability '{capability_name}', function '{name}' should not set response_data_type as {data_handler} unless an instance is configured in IntersectConfig.data_stores ." + f"On capability '{class_name}', function '{name}' should not set response_data_type as {data_handler} unless an instance is configured in IntersectConfig.data_stores ." ) docstring = inspect.cleandoc(method.__doc__) if method.__doc__ else None @@ -358,7 +362,7 @@ def _introspection_baseline( ) ): die( - f"On capability '{capability_name}', function '{name}' should have 'self' (unless a staticmethod) and zero or one additional parameters, and should not use keyword or variable length arguments (i.e. '*', *args, **kwargs)." + f"On capability '{class_name}', function '{name}' should have 'self' (unless a staticmethod) and zero or one additional parameters, and should not use keyword or variable length arguments (i.e. '*', *args, **kwargs)." ) # The schema format should be hard-coded and determined based on how Pydantic parses the schema. @@ -390,7 +394,7 @@ def _introspection_baseline( # Pydantic BaseModels require annotations even if using a default value, so we'll remain consistent. if annotation is inspect.Parameter.empty: die( - f"On capability '{capability_name}', parameter '{parameter.name}' type annotation on function '{name}' missing. {SCHEMA_HELP_MSG}" + f"On capability '{class_name}', parameter '{parameter.name}' type annotation on function '{name}' missing. {SCHEMA_HELP_MSG}" ) # rationale for disallowing function default values: # https://docs.pydantic.dev/latest/concepts/validation_decorator/#using-field-to-describe-function-arguments @@ -398,7 +402,7 @@ def _introspection_baseline( # also makes TypeAdapters considerably easier to construct if parameter.default is not inspect.Parameter.empty: die( - f"On capability '{capability_name}', parameter '{parameter.name}' should not use a default value in the function parameter (use 'typing_extensions.Annotated[TYPE, pydantic.Field(default=)]' instead - 'default_factory' is an acceptable, mutually exclusive argument to 'Field')." + f"On capability '{class_name}', parameter '{parameter.name}' should not use a default value in the function parameter (use 'typing_extensions.Annotated[TYPE, pydantic.Field(default=)]' instead - 'default_factory' is an acceptable, mutually exclusive argument to 'Field')." ) try: function_cache_request_adapter = TypeAdapter(annotation) @@ -410,7 +414,7 @@ def _introspection_baseline( ) except PydanticUserError as e: die( - f"On capability '{capability_name}', parameter '{parameter.name}' type annotation '{annotation}' on function '{name}' is invalid\n{e}" + f"On capability '{class_name}', parameter '{parameter.name}' type annotation '{annotation}' on function '{name}' is invalid\n{e}" ) else: @@ -419,7 +423,7 @@ def _introspection_baseline( # this block handles response parameters if return_annotation is inspect.Signature.empty: die( - f"On capability '{capability_name}', return type annotation on function '{name}' missing. {SCHEMA_HELP_MSG}" + f"On capability '{class_name}', return type annotation on function '{name}' missing. {SCHEMA_HELP_MSG}" ) try: function_cache_response_adapter = TypeAdapter(return_annotation) @@ -431,11 +435,12 @@ def _introspection_baseline( ) except PydanticUserError as e: die( - f"On capability '{capability_name}', return annotation '{return_annotation}' on function '{name}' is invalid.\n{e}" + f"On capability '{class_name}', return annotation '{return_annotation}' on function '{name}' is invalid.\n{e}" ) # final function mapping - function_map[name] = FunctionMetadata( + function_map[public_name] = FunctionMetadata( + type(capability), method, function_cache_request_adapter, function_cache_response_adapter, @@ -444,7 +449,7 @@ def _introspection_baseline( # this block handles events associated with intersect_messages (implies command pattern) function_events: dict[str, IntersectEventDefinition] = getattr(method, EVENT_ATTR_KEY) _add_events( - capability_name, + class_name, name, schemas, event_schemas, @@ -455,9 +460,9 @@ def _introspection_baseline( channels[name]['events'] = list(function_events.keys()) # parse global schemas - for capability_name, name, method, _ in event_funcs: + for class_name, name, method, _ in event_funcs: _add_events( - capability_name, + class_name, name, schemas, event_schemas, @@ -471,7 +476,9 @@ def _introspection_baseline( ) # this conditional allows for the status function to also be called like a message if status_fn_type_adapter and status_fn and status_fn_name: - function_map[status_fn_name] = FunctionMetadata( + public_status_name = f'{cap_name}.{status_fn_name}' + function_map[public_status_name] = FunctionMetadata( + type(capability), status_fn, None, status_fn_type_adapter, @@ -487,14 +494,15 @@ def _introspection_baseline( ) -def get_schema_and_functions_from_capability_implementation( - capability_type: type[IntersectBaseCapabilityImplementation], - capability_name: HierarchyConfig, +def get_schema_and_functions_from_capability_implementations( + capabilities: List[IntersectBaseCapabilityImplementation], + service_name: HierarchyConfig, excluded_data_handlers: set[IntersectDataHandler], ) -> tuple[ dict[str, Any], dict[str, FunctionMetadata], dict[str, EventMetadata], + IntersectBaseCapabilityImplementation | None, str | None, TypeAdapter[Any] | None, ]: @@ -502,20 +510,47 @@ def get_schema_and_functions_from_capability_implementation( In-depth introspection is handled later on. """ - ( - schemas, - (status_fn_name, status_schema, status_type_adapter), - channels, - function_map, - events, - event_map, - ) = _introspection_baseline(capability_type, excluded_data_handlers) + capability_type_docs : str = "" + status_function_cap : IntersectBaseCapabilityImplementation = None + status_function_name : str = None + status_function_schema : dict[str, Any] = None + status_function_adapter : TypeAdapter[Any] = None + schemas : dict[Any, Any] = dict() + channels : dict[str, dict[str, dict[str, Any]]] = dict() # endpoint schemas + function_map : dict[str, FunctionMetadata] = dict() # endpoint functionality + events : dict[str, Any] = dict() # event schemas + event_map : dict[str, EventMetadata] = dict() # event functionality + for capability in capabilities: + capability_type: type[IntersectBaseCapabilityImplementation] = type(capability) + if capability_type.__doc__: + capability_type_docs += inspect.cleandoc(capability_type.__doc__) + '\n' + ( + cap_schemas, + (cap_status_fn_name, cap_status_schema, cap_status_type_adapter), + cap_channels, + cap_function_map, + cap_events, + cap_event_map, + ) = _introspection_baseline(capability, excluded_data_handlers) + + if cap_status_fn_name and cap_status_schema and cap_status_type_adapter: + status_function_cap = capability + status_function_name = cap_status_fn_name + status_function_schema = cap_status_schema + status_function_adapter = cap_status_type_adapter + + schemas.update(cap_schemas) + channels.update(cap_channels) + function_map.update(cap_function_map) + events.update(cap_events) + event_map.update(cap_event_map) + asyncapi_spec = { 'asyncapi': ASYNCAPI_VERSION, 'x-intersect-version': version_string, 'info': { - 'title': capability_name.hierarchy_string('.'), + 'title': service_name.hierarchy_string('.'), 'version': '0.0.0', # NOTE: this will be modified by INTERSECT CORE, users do not manage their schema versions }, # applies to how an incoming message payload will be parsed. @@ -540,11 +575,12 @@ def get_schema_and_functions_from_capability_implementation( }, }, } - if capability_type.__doc__: - asyncapi_spec['info']['description'] = inspect.cleandoc(capability_type.__doc__) # type: ignore[index] - if status_schema: - asyncapi_spec['status'] = status_schema + if capability_type_docs != "": + asyncapi_spec['info']['description'] = capability_type_docs # type: ignore[index] + + if status_function_schema: + asyncapi_spec['status'] = status_function_schema """ TODO - might want to include these fields @@ -555,4 +591,4 @@ def get_schema_and_functions_from_capability_implementation( }, """ - return asyncapi_spec, function_map, event_map, status_fn_name, status_type_adapter + return asyncapi_spec, function_map, event_map, status_function_cap, status_function_name, status_function_adapter diff --git a/src/intersect_sdk/capability/base.py b/src/intersect_sdk/capability/base.py index a3c5143..e7b6716 100644 --- a/src/intersect_sdk/capability/base.py +++ b/src/intersect_sdk/capability/base.py @@ -26,6 +26,12 @@ def __init__(self) -> None: NOTE: If you write your own constructor, you MUST call super.__init__() inside of it. The Service will throw an error if you don't. """ + + self._capability_name : str = "InvalidCapability" + """ + The advertised name for the capability, as opposed to the implementation class name + """ + self.__intersect_sdk_observers__: list[IntersectEventObserver] = [] """ INTERNAL USE ONLY. @@ -43,6 +49,15 @@ def __init_subclass__(cls) -> None: ): msg = f"{cls.__name__}: Cannot override functions '_intersect_sdk_register_observer' or 'intersect_sdk_emit_event'" raise RuntimeError(msg) + + @property + def capability_name(self) -> str: + """The advertised name for the capability provided by this implementation""" + return self._capability_name + + @capability_name.setter + def capability_name(self, cname : str) -> str: + self._capability_name = cname @final def _intersect_sdk_register_observer(self, observer: IntersectEventObserver) -> None: diff --git a/src/intersect_sdk/client.py b/src/intersect_sdk/client.py index 332fdd5..4c53cfb 100644 --- a/src/intersect_sdk/client.py +++ b/src/intersect_sdk/client.py @@ -77,7 +77,7 @@ def __init__( Parameters: config: The IntersectClientConfig class - user_callback: The callback function you can use to handle response messages from the other Service. + user_callback: The callback function you can use to handle response messages from Services. If this is left empty, you can only send a single message event_callback: The callback function you can use to handle events from any Service. """ @@ -133,7 +133,7 @@ def __init__( if user_callback: # Do not persist, as we use the temporary client information to build this. self._control_plane_manager.add_subscription_channel( - f"{self._hierarchy.hierarchy_string('/')}/userspace", + f"{self._hierarchy.hierarchy_string('/')}/response", {self._handle_userspace_message_raw}, persist=False, ) @@ -429,12 +429,12 @@ def _send_userspace_message(self, params: IntersectClientMessageParams) -> None: """Send a userspace message, be it an initial message from the user or from the user's callback function.""" # ONE: SERIALIZE FUNCTION RESULTS # (function input should already be validated at this point) - response = GENERIC_MESSAGE_SERIALIZER.dump_json(params.payload, warnings=False) + msg_payload = GENERIC_MESSAGE_SERIALIZER.dump_json(params.payload, warnings=False) # TWO: SEND DATA TO APPROPRIATE DATA STORE try: - response_payload = self._data_plane_manager.outgoing_message_data_handler( - response, params.response_content_type, params.response_data_handler + out_payload = self._data_plane_manager.outgoing_message_data_handler( + msg_payload, params.content_type, params.data_handler ) except IntersectError: # NOTE @@ -447,17 +447,17 @@ def _send_userspace_message(self, params: IntersectClientMessageParams) -> None: msg = create_userspace_message( source=self._hierarchy.hierarchy_string('.'), destination=params.destination, - content_type=params.response_content_type, - data_handler=params.response_data_handler, + content_type=params.content_type, + data_handler=params.data_handler, operation_id=params.operation, - payload=response_payload, + payload=out_payload, ) logger.debug(f'Send userspace message:\n{msg}') - response_channel = f"{params.destination.replace('.', '/')}/userspace" + channel = f"{params.destination.replace('.', '/')}/request" # WARNING: If both the Service and the Client drop, the Service will execute the command # but cannot communicate the response to the Client. # in experiment controllers or production, you'll want to set persist to True - self._control_plane_manager.publish_message(response_channel, msg, persist=False) + self._control_plane_manager.publish_message(channel, msg, persist=False) # TODO - consider removing this entire concept def _heartbeat_ticker(self) -> None: diff --git a/src/intersect_sdk/client_callback_definitions.py b/src/intersect_sdk/client_callback_definitions.py index 6b3ee2b..5ffe2db 100644 --- a/src/intersect_sdk/client_callback_definitions.py +++ b/src/intersect_sdk/client_callback_definitions.py @@ -33,14 +33,14 @@ class IntersectClientMessageParams: If you want to just use the service's default value for a request (assuming it has a default value for a request), you may set this as None. """ - response_content_type: IntersectMimeType = IntersectMimeType.JSON + content_type: IntersectMimeType = IntersectMimeType.JSON """ - The IntersectMimeType of your response. You'll want this to match with the ContentType of the function from the schema. + The IntersectMimeType of your message. You'll want this to match with the ContentType of the function from the schema. default: IntersectMimeType.JSON """ - response_data_handler: IntersectDataHandler = IntersectDataHandler.MESSAGE + data_handler: IntersectDataHandler = IntersectDataHandler.MESSAGE """ The IntersectDataHandler you want to use (most people can just use IntersectDataHandler.MESSAGE here, unless your data is very large) diff --git a/src/intersect_sdk/schema.py b/src/intersect_sdk/schema.py index 77cd583..313a318 100644 --- a/src/intersect_sdk/schema.py +++ b/src/intersect_sdk/schema.py @@ -33,7 +33,7 @@ Any, ) -from ._internal.schema import get_schema_and_functions_from_capability_implementation +from ._internal.schema import get_schema_and_functions_from_capability_implementations if TYPE_CHECKING: from .capability.base import IntersectBaseCapabilityImplementation @@ -42,13 +42,13 @@ def get_schema_from_capability_implementation( capability_type: type[IntersectBaseCapabilityImplementation], - capability_name: HierarchyConfig, + service_name: HierarchyConfig, ) -> dict[str, Any]: """The goal of this function is to be able to generate a complete schema matching the AsyncAPI spec 2.6.0 from a BaseModel class. Params: - capability_type - the SDK user will provide the class of their capability handler, which generates the schema - - capability_name - ideally, this could be scanned by the package name. Meant to be descriptive, i.e. "nionswift" + - service_name - ideally, this could be scanned by the package name. Meant to be descriptive, i.e. "nionswift" SOME NOTES ABOUT THE SCHEMA @@ -69,9 +69,10 @@ def get_schema_from_capability_implementation( - Channel names just mimic the function names for now """ - schemas, _, _, _, _ = get_schema_and_functions_from_capability_implementation( - capability_type, - capability_name, + cap_instance = capability_type() + schemas, _, _, _, _, _ = get_schema_and_functions_from_capability_implementations( + [cap_instance], + service_name, set(), # assume all data handlers are configured if user is just checking their schema ) return schemas diff --git a/src/intersect_sdk/service.py b/src/intersect_sdk/service.py index ec67099..220edfe 100644 --- a/src/intersect_sdk/service.py +++ b/src/intersect_sdk/service.py @@ -16,9 +16,11 @@ from __future__ import annotations +from datetime import datetime, timezone +from threading import Condition, Lock from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Callable -from uuid import uuid4 +from typing import TYPE_CHECKING, Any, Callable, List, Tuple +from uuid import uuid1, uuid3, UUID from pydantic import ValidationError from pydantic_core import PydanticSerializationError @@ -30,7 +32,10 @@ SHUTDOWN_KEYS, STRICT_VALIDATION, ) -from ._internal.control_plane.control_plane_manager import ControlPlaneManager +from ._internal.control_plane.control_plane_manager import ( + ControlPlaneManager, + GENERIC_MESSAGE_SERIALIZER +) from ._internal.data_plane.data_plane_manager import DataPlaneManager from ._internal.exceptions import IntersectApplicationError, IntersectError from ._internal.interfaces import IntersectEventObserver @@ -42,11 +47,12 @@ create_userspace_message, deserialize_and_validate_userspace_message, ) -from ._internal.schema import get_schema_and_functions_from_capability_implementation +from ._internal.schema import get_schema_and_functions_from_capability_implementations from ._internal.stoppable_thread import StoppableThread from ._internal.utils import die from ._internal.version_resolver import resolve_user_version from .capability.base import IntersectBaseCapabilityImplementation +from .client_callback_definitions import IntersectClientMessageParams from .config.service import IntersectServiceConfig from .core_definitions import IntersectDataHandler, IntersectMimeType from .version import version_string @@ -64,67 +70,97 @@ class IntersectService(IntersectEventObserver): you return your output. The service automatically integrates all of the following components: - - The user-defined capability + - The user-defined capabilities - Any message brokers - Any core INTERSECT data layers What it does NOT do: - deal with any custom messaging logic - i.e. Pyro logic, an internal ZeroMQ system, etc. ... these should be defined on the capability level. - - deal with any application logic - that should be handled by the user's capability + - deal with any application logic - that should be handled by the capabilities - Users should generally not need to interact with objects of this class outside of the constructor and the startup() and shutdown() functions. It's advisable - not to mutate the service object yourself, though you can freely log out properties for debugging purposes. + Users should generally not need to interact with objects of this class outside of the constructor and the following functions: startup(), add_startup_messages(), shutdown(), add_shutdown_messages(), create_external_request(). + It's advisable not to mutate the service object yourself, though you can freely log out properties for debugging purposes. Note: The ONLY stable function methods are: - the constructor - startup() + - add_startup_messages() - shutdown() + - add_shutdown_messages() - is_connected() - forbid_keys() - allow_keys() - allow_all_functions() - get_blocked_keys() + - create_external_request() No other functions or parameters are guaranteed to remain stable. """ + class ExternalRequest: + request_id : UUID + request_name : str + cv : Condition + processed : bool = False + error : str = None + request : Any = None + response : Any = None + response_fn : Callable = None + waiting : bool = False + + def __init__(self, req_id : UUID, req_name : str) -> None: + self.cv = Condition() + self.request_id = req_id + self.request_name = req_name + + def __init__( self, - capability: IntersectBaseCapabilityImplementation, + capabilities: List[IntersectBaseCapabilityImplementation], config: IntersectServiceConfig, ) -> None: """The constructor performs almost all validation checks necessary to function in the INTERSECT ecosystem, with the exception of checking connections/credentials to any backing services. Parameters: - capability: Your capability implementation class + capabilities: Your list of capability implementation classes config: The IntersectConfig class """ - if not isinstance(capability, IntersectBaseCapabilityImplementation): - die( - f'IntersectService parameter must inherit from intersect_sdk.IntersectBaseCapabilityImplementation instead of "{capability.__class__.__name__}" .' - ) - if not hasattr(capability, '__intersect_sdk_observers__'): - die( - f'{capability.__class__.__name__} needs to call "super().__init__()" in the constructor.' - ) + + for cap in capabilities: + if not isinstance(cap, IntersectBaseCapabilityImplementation): + die( + f'IntersectService parameter must inherit from intersect_sdk.IntersectBaseCapabilityImplementation instead of "{cap.__class__.__name__}" .' + ) + if not hasattr(cap, '__intersect_sdk_observers__'): + die( + f'{cap.__class__.__name__} needs to call "super().__init__()" in the constructor.' + ) + + # we generally start observing and don't stop, doesn't really matter if we startup or shutdown + cap._intersect_sdk_register_observer(self) # noqa: SLF001 (we don't want users calling or overriding it, but this is fine.) + + self.capabilities = capabilities + # this is called here in case a user created the object using "IntersectServiceConfig.model_construct()" to skip validation config = IntersectServiceConfig.model_validate(config) - self.capability = capability + ( schema, function_map, event_map, + status_fn_capability, status_fn_name, - status_type_adapter, - ) = get_schema_and_functions_from_capability_implementation( - capability.__class__, - capability_name=config.hierarchy, + status_type_adapter + ) = get_schema_and_functions_from_capability_implementations( + self.capabilities, + service_name=config.hierarchy, excluded_data_handlers=config.data_stores.get_missing_data_store_types(), ) self._schema = schema """ Stringified schema of the user's application. Gets sent in several status message requests. """ + self._function_map = MappingProxyType(function_map) """ INTERNAL USE ONLY @@ -134,6 +170,7 @@ def __init__( You can get user-defined properties from the method via getattr(_function_map.method, KEY), the keys get set in the intersect_message decorator function (annotations.py). """ + self._event_map = MappingProxyType(event_map) """ INTERNAL USE ONLY @@ -153,13 +190,14 @@ def __init__( """ self._hierarchy = config.hierarchy + self._uuid = uuid3(uuid1(), config.hierarchy.hierarchy_string('.')) self._status_thread: StoppableThread | None = None self._status_ticker_interval = config.status_interval self._status_retrieval_fn: Callable[[], bytes] = ( ( lambda: status_type_adapter.dump_json( - getattr(self.capability, status_fn_name)(), by_alias=True, warnings='error' + getattr(status_fn_capability, status_fn_name)(), by_alias=True, warnings='error' ) ) if status_type_adapter and status_fn_name @@ -168,25 +206,47 @@ def __init__( self._status_memo = self._status_retrieval_fn() + self._external_request_thread = None + self._external_requests_lock = Lock() + self._external_requests = dict() + self._external_request_ctr = 0 + + self._startup_messages : List[ Tuple[IntersectClientMessageParams, Callable] ] = list() + self._resend_startup_messages = True + self._sent_startup_messages = False + + self._shutdown_messages : List[ Tuple[IntersectClientMessageParams, Callable] ] = list() + self._data_plane_manager = DataPlaneManager(self._hierarchy, config.data_stores) # we PUBLISH messages on this channel self._lifecycle_channel_name = f"{config.hierarchy.hierarchy_string('/')}/lifecycle" # we PUBLISH event messages on this channel self._events_channel_name = f"{config.hierarchy.hierarchy_string('/')}/events" - # we SUBSCRIBE to messages on this channel - self._userspace_channel_name = f"{config.hierarchy.hierarchy_string('/')}/userspace" + # we SUBSCRIBE to messages on this channel to receive requests + self._service_channel_name = f"{config.hierarchy.hierarchy_string('/')}/request" + # we SUBSCRIBE to messages on this channel to receive responses + self._client_channel_name = f"{config.hierarchy.hierarchy_string('/')}/response" + self._control_plane_manager = ControlPlaneManager( control_configs=config.brokers, ) # our userspace queue should be able to survive shutdown self._control_plane_manager.add_subscription_channel( - self._userspace_channel_name, - {self._handle_userspace_message_raw}, - persist=True, + self._service_channel_name, + {self._handle_service_message_raw}, + persist=True + ) + self._control_plane_manager.add_subscription_channel( + self._client_channel_name, + {self._handle_client_message_raw}, + persist=False ) - # we generally start observing and don't stop, doesn't really matter if we startup or shutdown - self.capability._intersect_sdk_register_observer(self) # noqa: SLF001 (we don't want users calling or overriding it, but this is fine.) + def _get_capability(self, target : str) -> Any | None: + for cap in self.capabilities: + if cap.capability_name == target: + return cap + return None @final def startup(self) -> Self: @@ -219,10 +279,27 @@ def startup(self) -> Self: # Start the status thread if it doesn't already exist if self._status_thread is None: self._status_thread = StoppableThread( - target=self._status_ticker, name=f'IntersectService_{uuid4()!s}_status_thread' + target=self._status_ticker, name=f'IntersectService_{self._uuid}_status_thread' ) self._status_thread.start() + # Process pending startup messages + if self._resend_startup_messages or not self._sent_startup_messages: + logger.info('Sending startup messages') + for tup in self._startup_messages: + message, fn = tup + self.create_external_request(request=message, + response_handler=fn) + self.process_external_requests() + self._sent_startup_messages = True + + # Start the external request thread if it doesn't already exist + if self._external_request_thread is None: + self._external_request_thread = StoppableThread( + target=self._send_external_requests, name=f'IntersectService_{self._uuid}_ext_req_thread' + ) + self._external_request_thread.start() + logger.info('Service startup complete') return self @@ -243,6 +320,18 @@ def shutdown(self, reason: str | None = None) -> Self: """ logger.info(f'Service is shutting down (reason: {reason})') + if self._external_request_thread is not None: + self._external_request_thread.stop() + self._external_request_thread.join() + self._external_request_thread = None + + logger.info('Sending shutdown messages') + for tup in self._shutdown_messages: + message, fn = tup + self.create_external_request(request=message, + response_handler=fn) + self.process_external_requests() + # Stop polling if self._status_thread is not None: self._status_thread.stop() @@ -348,7 +437,92 @@ def get_blocked_keys(self) -> set[str]: """ return self._function_keys.copy() - def _handle_userspace_message_raw(self, raw: bytes) -> None: + def add_startup_messages(self, messages : List[Tuple[IntersectClientMessageParams, Callable]]) -> None: + self._startup_messages.extend(messages) + + def add_shutdown_messages(self, messages : List[Tuple[IntersectClientMessageParams, Callable]]) -> None: + self._shutdown_messages.extend(messages) + + def _new_external_request(self) -> IntersectService.ExternalRequest: + self._external_request_ctr += 1 + request_name = f'ext-req-{self._external_request_ctr}' + request_uuid = uuid3(self._uuid, request_name) + req = IntersectService.ExternalRequest(req_id=request_uuid, + req_name=request_name) + self._external_requests_lock.acquire_lock(blocking=True) + self._external_requests[str(request_uuid)] = req + self._external_requests_lock.release_lock() + return req + + def _delete_external_request(self, req_id : UUID) -> None: + req_id_str = str(req_id) + if req_id_str in self._external_requests: + req : IntersectService.ExternalRequest = self._external_requests.pop(req_id_str) + del req + + def _get_external_request(self, req_id : UUID) -> IntersectService.ExternalRequest: + req_id_str = str(req_id) + if req_id_str in self._external_requests: + req : IntersectService.ExternalRequest = self._external_requests[req_id_str] + return req + return None + + def create_external_request(self, + request: IntersectClientMessageParams, + response_handler : Callable = None) -> UUID: + # create an external request structure with a Condition we can wait on + extreq : IntersectService.ExternalRequest = self._new_external_request() + extreq.request = request + extreq.response_fn = response_handler + return extreq.request_id + + def process_external_requests(self) -> None: + self._external_requests_lock.acquire_lock(blocking=True) + for extreq in self._external_requests.values(): + if not extreq.processed: + self._process_external_request(extreq) + self._external_requests_lock.release_lock() + + def _process_external_request(self, extreq: IntersectService.ExternalRequest) -> None: + response = None + cleanup_req = False + + now = datetime.now(timezone.utc) + logger.debug(f'Processing external request {extreq.request_id} @ {now}') + + with extreq.cv: + # execute the request + extreq.processed = True + if self._send_client_message(request_id=extreq.request_id, params=extreq.request): + # MJB NOTE: currently it is impossible to get a response for the + # external request when this function is called while + # handling an incoming request, so we are just ignoring + # any wait timeouts below. + + # wait on the response condition and get the response + extreq.waiting = True + if extreq.cv.wait(timeout=1.0): + if extreq.error is None: + response = extreq.response + else: + error_msg = extreq.error + logger.warning(f'External service request encountered an error: {error_msg}') + cleanup_req = True + else: + logger.debug('Request wait timed-out!') + extreq.waiting = False + else: + logger.warning('Failed to send request!') + + # process the response + if response is not None: + if extreq.response_fn is not None: + extreq.response_fn(response) + + if cleanup_req: + self._delete_external_request(extreq.request_name) + + def _handle_service_message_raw(self, raw: bytes) -> None: """Main broker callback function. Deserializes and validates a userspace message from a broker. @@ -358,14 +532,14 @@ def _handle_userspace_message_raw(self, raw: bytes) -> None: try: message = deserialize_and_validate_userspace_message(raw) logger.debug(f'Received userspace message:\n{message}') - response_msg = self._handle_userspace_message(message) + response_msg = self._handle_service_message(message) if response_msg: logger.debug( 'Send %s message:\n%s', 'error' if response_msg['headers']['has_error'] else 'userspace', response_msg, ) - response_channel = f"{message['headers']['source'].replace('.', '/')}/userspace" + response_channel = f"{message['headers']['source'].replace('.', '/')}/response" # Persistent userspace messages may be useful for orchestration. # Persistence will not hurt anything. self._control_plane_manager.publish_message( @@ -376,7 +550,7 @@ def _handle_userspace_message_raw(self, raw: bytes) -> None: f'Invalid message received on userspace message channel, ignoring. Full message:\n{e}' ) - def _handle_userspace_message(self, message: UserspaceMessage) -> UserspaceMessage | None: + def _handle_service_message(self, message: UserspaceMessage) -> UserspaceMessage | None: """Main logic for handling a userspace message, minus all broker logic. Params @@ -405,6 +579,13 @@ def _handle_userspace_message(self, message: UserspaceMessage) -> UserspaceMessa err_msg = f"Function '{operation}' is currently not available for use." logger.error(err_msg) return self._make_error_message(err_msg, message) + + operation_capability, operation_method = operation.split('.') + target_capability = self._get_capability(operation_capability) + if target_capability is None: + err_msg = f"Could not locate service capability providing '{operation_capability}' for operation {operation}." + logger.error(err_msg) + return self._make_error_message(err_msg, message) # THREE: GET DATA FROM APPROPRIATE DATA STORE try: @@ -416,7 +597,7 @@ def _handle_userspace_message(self, message: UserspaceMessage) -> UserspaceMessa try: # FOUR: CALL USER FUNCTION AND GET MESSAGE - response = self._call_user_function(operation, operation_meta, request_params) + response = self._call_user_function(target_capability, operation_method, operation_meta, request_params) # FIVE: SEND DATA TO APPROPRIATE DATA STORE response_data_handler = getattr(operation_meta.method, RESPONSE_DATA) response_content_type = getattr(operation_meta.method, RESPONSE_CONTENT) @@ -443,10 +624,85 @@ def _handle_userspace_message(self, message: UserspaceMessage) -> UserspaceMessa data_handler=response_data_handler, operation_id=message['operationId'], payload=response_payload, + message_id=message['messageId'] # associate response with request ) + + def _handle_client_message_raw(self, raw: bytes) -> None: + """Broker callback, deserialize and validate a userspace message from a broker.""" + try: + message = deserialize_and_validate_userspace_message(raw) + logger.debug(f'Received userspace message:\n{message}') + self._handle_client_message(message) + except ValidationError as e: + logger.warning( + f'Invalid message received on client message channel, ignoring. Full message:\n{e}' + ) + + def _handle_client_message(self, message: UserspaceMessage) -> None: + """Handle a deserialized userspace message.""" + + extreq = self._get_external_request(message['messageId']) + if extreq is not None: + error_msg : str = None + try: + msg_payload = GENERIC_MESSAGE_SERIALIZER.validate_json( + self._data_plane_manager.incoming_message_data_handler(message) + ) + except ValidationError as e: + error_msg = f'Service sent back invalid response:\n{e}' + logger.warning(error_msg) + except IntersectError: + error_msg = f'INTERNAL ERROR: failed to get message payload from data handler' + logger.error(error_msg) + + with extreq.cv: + if error_msg is not None: + extreq.error = error_msg + else: + extreq.response = msg_payload + if extreq.waiting: + extreq.cv.notify() + else: + error_msg = f'No external request found for message:\n{message}' + logger.warning(error_msg) + + def _send_client_message(self, request_id : UUID, params: IntersectClientMessageParams) -> bool: + """Send a userspace message.""" + # ONE: VALIDATE AND SERIALIZE FUNCTION RESULTS + try: + params = IntersectClientMessageParams.model_validate(params) + except ValidationError as e: + logger.error(f'Invalid message parameters:\n{e}') + return False + request = GENERIC_MESSAGE_SERIALIZER.dump_json(params.payload, warnings=False) + + # TWO: SEND DATA TO APPROPRIATE DATA STORE + try: + request_payload = self._data_plane_manager.outgoing_message_data_handler( + request, params.content_type, params.data_handler + ) + except IntersectError: + return False + + # THREE: SEND MESSAGE + msg = create_userspace_message( + source=self._hierarchy.hierarchy_string('.'), + destination=params.destination, + service_version=self._version, + content_type=params.content_type, + data_handler=params.data_handler, + operation_id=params.operation, + payload=request_payload, + message_id=request_id + ) + logger.debug(f'Sending client message:\n{msg}') + request_channel = f"{params.destination.replace('.', '/')}/request" + self._control_plane_manager.publish_message(request_channel, msg) + return True def _call_user_function( self, + fn_cap: IntersectBaseCapabilityImplementation, fn_name: str, fn_meta: FunctionMetadata, fn_params: bytes, @@ -456,6 +712,7 @@ def _call_user_function( Basic validations defined from a user's type definitions will also occur here. Params + fn_cap = capability implementing the user function fn_name = operation. These get represented in the schema as "channels". fn_meta = all information stored about the user's operation. This includes user-defined params and the request/response (de)serializers. fn_params = the request argument. @@ -501,7 +758,7 @@ def _call_user_function( logger.warning(err_msg) raise try: - response = getattr(self.capability, fn_name)(request_obj) + response = getattr(fn_cap, fn_name)(request_obj) except ( Exception ) as e: # (need to catch all possible exceptions to gracefully handle the thread) @@ -509,7 +766,7 @@ def _call_user_function( raise IntersectApplicationError from e else: try: - response = getattr(self.capability, fn_name)() + response = getattr(fn_cap, fn_name)() except ( Exception ) as e: # (need to catch all possible exceptions to gracefully handle the thread) @@ -587,6 +844,7 @@ def _make_error_message( data_handler=IntersectDataHandler.MESSAGE, operation_id=original_message['operationId'], payload=error_string, + message_id=original_message['messageId'], # associate error reply with original has_error=True, ) @@ -638,3 +896,13 @@ def _status_ticker(self) -> None: payload={'schema': self._schema, 'status': self._status_memo}, ) self._status_thread.wait(self._status_ticker_interval) + + def _send_external_requests(self) -> None: + """Periodically sends messages added to self._external_messages. Runs in a separate thread.""" + # initial wait should guarantee that first request message does not beat initial startup message + if self._external_request_thread: + self._external_request_thread.wait(10.0) + while not self._external_request_thread.stopped(): + self.process_external_requests() + self._external_request_thread.wait(0.5) + diff --git a/tests/e2e/test_examples.py b/tests/e2e/test_examples.py index 7d9d679..c6af75f 100644 --- a/tests/e2e/test_examples.py +++ b/tests/e2e/test_examples.py @@ -82,13 +82,13 @@ def test_example_2_counter(): == 'Source: "counting-organization.counting-facility.counting-system.counting-subsystem.counting-service"' ) # test operation - assert lines[1] == 'Operation: "start_count"' - assert lines[5] == 'Operation: "stop_count"' - assert lines[9] == 'Operation: "start_count"' - assert lines[13] == 'Operation: "reset_count"' - assert lines[17] == 'Operation: "reset_count"' - assert lines[21] == 'Operation: "start_count"' - assert lines[25] == 'Operation: "stop_count"' + assert lines[1] == 'Operation: "CountingExample.start_count"' + assert lines[5] == 'Operation: "CountingExample.stop_count"' + assert lines[9] == 'Operation: "CountingExample.start_count"' + assert lines[13] == 'Operation: "CountingExample.reset_count"' + assert lines[17] == 'Operation: "CountingExample.reset_count"' + assert lines[21] == 'Operation: "CountingExample.start_count"' + assert lines[25] == 'Operation: "CountingExample.stop_count"' # test payloads # if 'count' is within 3 steps of the subtrahend, just pass the test diff --git a/tests/fixtures/example_schema.json b/tests/fixtures/example_schema.json index 99e4b7d..d923449 100644 --- a/tests/fixtures/example_schema.json +++ b/tests/fixtures/example_schema.json @@ -4,7 +4,7 @@ "info": { "title": "test.test.test.test.test", "version": "0.0.0", - "description": "This is an example of the overarching capability class a user creates that we want to inject into the service.\n\nWhen defining entrypoints to your capability, use the @intersect_message() annotation. Your class will need\nat least one function with this annotation. These functions REQUIRE type annotations to function properly.\nSee the @intersect_message() annotation for more information.\n\nYou can potentially extend from multiple preexisting Capabilities in this class - each Capability may have\nseveral abstract functions which would need to be implemented by the user.\n\nBeyond this, you may define your capability class however you like, including through its constructor." + "description": "This is an example of the overarching capability class a user creates that we want to inject into the service.\n\nWhen defining entrypoints to your capability, use the @intersect_message() annotation. Your class will need\nat least one function with this annotation. These functions REQUIRE type annotations to function properly.\nSee the @intersect_message() annotation for more information.\n\nYou can potentially extend from multiple preexisting Capabilities in this class - each Capability may have\nseveral abstract functions which would need to be implemented by the user.\n\nBeyond this, you may define your capability class however you like, including through its constructor.\n" }, "defaultContentType": "application/json", "channels": { diff --git a/tests/fixtures/example_schema.py b/tests/fixtures/example_schema.py index 37b2df8..26c2f92 100644 --- a/tests/fixtures/example_schema.py +++ b/tests/fixtures/example_schema.py @@ -235,6 +235,7 @@ def __init__(self) -> None: which handles talking to the various INTERSECT-related backing services. """ super().__init__() + self.capability_name = "DummyCapability" self._status_example = DummyStatus( functions_called=0, last_function_called='', diff --git a/tests/integration/test_return_type_mismatch.py b/tests/integration/test_return_type_mismatch.py index 5d99a69..e4a468b 100644 --- a/tests/integration/test_return_type_mismatch.py +++ b/tests/integration/test_return_type_mismatch.py @@ -44,8 +44,10 @@ def wrong_return_annotation(self, param: int) -> int: def make_intersect_service() -> IntersectService: + capability = ReturnTypeMismatchCapabilityImplementation() + capability.capability_name = "ReturnTypeMismatchCapability" return IntersectService( - ReturnTypeMismatchCapabilityImplementation(), + [capability], IntersectServiceConfig( hierarchy=FAKE_HIERARCHY_CONFIG, data_stores=DataStoreConfigMap( @@ -96,19 +98,19 @@ def userspace_msg_callback(payload: bytes) -> None: msg[0] = deserialize_and_validate_userspace_message(payload) message_interceptor.add_subscription_channel( - 'msg/msg/msg/msg/msg/userspace', {userspace_msg_callback}, False + 'msg/msg/msg/msg/msg/response', {userspace_msg_callback}, False ) message_interceptor.connect() intersect_service.startup() time.sleep(1.0) message_interceptor.publish_message( - intersect_service._userspace_channel_name, + intersect_service._service_channel_name, create_userspace_message( source='msg.msg.msg.msg.msg', destination='test.test.test.test.test', content_type=IntersectMimeType.JSON, data_handler=IntersectDataHandler.MESSAGE, - operation_id='wrong_return_annotation', + operation_id='ReturnTypeMismatchCapability.wrong_return_annotation', # calculate_fibonacci takes in a tuple of two integers but we'll just send it one payload=b'2', ), diff --git a/tests/integration/test_service.py b/tests/integration/test_service.py index 3cf7a52..e8a8ecd 100644 --- a/tests/integration/test_service.py +++ b/tests/integration/test_service.py @@ -39,7 +39,7 @@ def make_intersect_service() -> IntersectService: return IntersectService( - DummyCapabilityImplementation(), + [DummyCapabilityImplementation()], IntersectServiceConfig( hierarchy=FAKE_HIERARCHY_CONFIG, data_stores=DataStoreConfigMap( @@ -92,13 +92,16 @@ def test_control_plane_connections(): assert intersect_service.is_connected() is False channels = intersect_service._control_plane_manager.get_subscription_channels() - # we have one channel (even if we're disconnected) ... - assert len(channels) == 1 - # ... and one callback function for this channel - channel_key = next(iter(channels)) - assert len(channels[channel_key].callbacks) == 1 - - intersect_service._control_plane_manager.remove_subscription_channel(channel_key) + # we have two channels (even if we're disconnected) ... + assert len(channels) == 2 + # ... and one callback function for each channel + channel_keys = [] + for channel_key in iter(channels): + channel_keys.append(channel_key) + assert len(channels[channel_key].callbacks) == 1 + + for channel_key in channel_keys: + intersect_service._control_plane_manager.remove_subscription_channel(channel_key) assert len(intersect_service._control_plane_manager.get_subscription_channels()) == 0 @@ -112,19 +115,19 @@ def userspace_msg_callback(payload: bytes) -> None: msg[0] = deserialize_and_validate_userspace_message(payload) message_interceptor.add_subscription_channel( - 'msg/msg/msg/msg/msg/userspace', {userspace_msg_callback}, False + 'msg/msg/msg/msg/msg/response', {userspace_msg_callback}, False ) message_interceptor.connect() intersect_service.startup() time.sleep(1.0) message_interceptor.publish_message( - intersect_service._userspace_channel_name, + intersect_service._service_channel_name, create_userspace_message( source='msg.msg.msg.msg.msg', destination='test.test.test.test.test', content_type=IntersectMimeType.JSON, data_handler=IntersectDataHandler.MESSAGE, - operation_id='calculate_fibonacci', + operation_id='DummyCapability.calculate_fibonacci', payload=b'[4,6]', ), True, @@ -147,19 +150,19 @@ def userspace_msg_callback(payload: bytes) -> None: msg[0] = deserialize_and_validate_userspace_message(payload) message_interceptor.add_subscription_channel( - 'msg/msg/msg/msg/msg/userspace', {userspace_msg_callback}, False + 'msg/msg/msg/msg/msg/response', {userspace_msg_callback}, False ) message_interceptor.connect() intersect_service.startup() time.sleep(1.0) message_interceptor.publish_message( - intersect_service._userspace_channel_name, + intersect_service._service_channel_name, create_userspace_message( source='msg.msg.msg.msg.msg', destination='test.test.test.test.test', content_type=IntersectMimeType.JSON, data_handler=IntersectDataHandler.MESSAGE, - operation_id='test_generator', + operation_id='DummyCapability.test_generator', payload=b'"res"', ), True, @@ -181,19 +184,19 @@ def userspace_msg_callback(payload: bytes) -> None: msg[0] = deserialize_and_validate_userspace_message(payload) message_interceptor.add_subscription_channel( - 'msg/msg/msg/msg/msg/userspace', {userspace_msg_callback}, False + 'msg/msg/msg/msg/msg/response', {userspace_msg_callback}, False ) message_interceptor.connect() intersect_service.startup() time.sleep(1.0) message_interceptor.publish_message( - intersect_service._userspace_channel_name, + intersect_service._service_channel_name, create_userspace_message( source='msg.msg.msg.msg.msg', destination='test.test.test.test.test', content_type=IntersectMimeType.JSON, data_handler=IntersectDataHandler.MESSAGE, - operation_id='valid_default_argument', + operation_id='DummyCapability.valid_default_argument', payload=b'null', # if sending null as the payload, the SDK will call the function's default value ), True, @@ -216,19 +219,19 @@ def userspace_msg_callback(payload: bytes) -> None: msg[0] = deserialize_and_validate_userspace_message(payload) message_interceptor.add_subscription_channel( - 'msg/msg/msg/msg/msg/userspace', {userspace_msg_callback}, False + 'msg/msg/msg/msg/msg/response', {userspace_msg_callback}, False ) message_interceptor.connect() intersect_service.startup() time.sleep(1.0) message_interceptor.publish_message( - intersect_service._userspace_channel_name, + intersect_service._service_channel_name, create_userspace_message( source='msg.msg.msg.msg.msg', destination='test.test.test.test.test', content_type=IntersectMimeType.JSON, data_handler=IntersectDataHandler.MESSAGE, - operation_id='calculate_fibonacci', + operation_id='DummyCapability.calculate_fibonacci', # calculate_fibonacci takes in a tuple of two integers but we'll just send it one payload=b'[2]', ), @@ -255,20 +258,19 @@ def userspace_msg_callback(payload: bytes) -> None: msg[0] = deserialize_and_validate_userspace_message(payload) message_interceptor.add_subscription_channel( - 'msg/msg/msg/msg/msg/userspace', {userspace_msg_callback}, False + 'msg/msg/msg/msg/msg/response', {userspace_msg_callback}, False ) message_interceptor.connect() intersect_service.startup() time.sleep(1.0) message_interceptor.publish_message( - intersect_service._userspace_channel_name, + intersect_service._service_channel_name, create_userspace_message( source='msg.msg.msg.msg.msg', destination='test.test.test.test.test', content_type=IntersectMimeType.JSON, data_handler=IntersectDataHandler.MESSAGE, - operation_id='THIS_FUNCTION_DOES_NOT_EXIST', - # calculate_fibonacci takes in a tuple of two integers but we'll just send it one + operation_id='DummyCapability.THIS_FUNCTION_DOES_NOT_EXIST', payload=b'null', ), True, @@ -292,19 +294,19 @@ def userspace_msg_callback(payload: bytes) -> None: msg[0] = deserialize_and_validate_userspace_message(payload) message_interceptor.add_subscription_channel( - 'msg/msg/msg/msg/msg/userspace', {userspace_msg_callback}, False + 'msg/msg/msg/msg/msg/response', {userspace_msg_callback}, False ) message_interceptor.connect() intersect_service.startup() time.sleep(1.0) message_interceptor.publish_message( - intersect_service._userspace_channel_name, + intersect_service._service_channel_name, create_userspace_message( source='msg.msg.msg.msg.msg', destination='test.test.test.test.test', content_type=IntersectMimeType.JSON, data_handler=IntersectDataHandler.MESSAGE, - operation_id='test_datetime', + operation_id='DummyCapability.test_datetime', payload=b'"1970-01-01T00:00:00Z"', ), True, @@ -339,7 +341,7 @@ def lifecycle_msg_callback(payload: bytes) -> None: 'test/test/test/test/test/lifecycle', {lifecycle_msg_callback}, False ) # we do not really care about the userspace message response, but we'll listen to it to consume it - message_interceptor.add_subscription_channel('msg/msg/msg/msg/msg/userspace', set(), False) + message_interceptor.add_subscription_channel('msg/msg/msg/msg/msg/response', set(), False) message_interceptor.connect() # sleep a moment to make sure message_interceptor catches the startup message time.sleep(1.0) @@ -349,13 +351,13 @@ def lifecycle_msg_callback(payload: bytes) -> None: # send a message to trigger a status update (just the way the example service's domain works, not intrinsic) message_interceptor.publish_message( - intersect_service._userspace_channel_name, + intersect_service._service_channel_name, create_userspace_message( source='msg.msg.msg.msg.msg', destination='test.test.test.test.test', content_type=IntersectMimeType.JSON, data_handler=IntersectDataHandler.MESSAGE, - operation_id='verify_float_dict', + operation_id='DummyCapability.verify_float_dict', # note that the dict key MUST be a string, even though the input wants a float key payload=b'{"1.2":"one point two"}', ), diff --git a/tests/unit/test_invalid_schema_runtime.py b/tests/unit/test_invalid_schema_runtime.py index a6fd00c..8d5430f 100644 --- a/tests/unit/test_invalid_schema_runtime.py +++ b/tests/unit/test_invalid_schema_runtime.py @@ -38,5 +38,5 @@ def test_minio_not_allowed_without_config(caplog: pytest.LogCaptureFixture): ], ) with pytest.raises(SystemExit): - IntersectService(cap, conf) + IntersectService([cap], conf) assert "function 'arbitrary_function' should not set response_data_type as 1" in caplog.text diff --git a/tests/unit/test_schema_valid.py b/tests/unit/test_schema_valid.py index 44bffc4..aa238ea 100644 --- a/tests/unit/test_schema_valid.py +++ b/tests/unit/test_schema_valid.py @@ -8,7 +8,7 @@ RESPONSE_DATA, STRICT_VALIDATION, ) -from intersect_sdk._internal.schema import get_schema_and_functions_from_capability_implementation +from intersect_sdk._internal.schema import get_schema_and_functions_from_capability_implementations from intersect_sdk.schema import get_schema_from_capability_implementation from tests.fixtures.example_schema import ( @@ -37,43 +37,46 @@ def test_schema_comparison(): def test_verify_status_fn(): + dummy_cap = DummyCapabilityImplementation() ( schema, function_map, _, + status_fn_capability, status_fn_name, - status_type_adapter, - ) = get_schema_and_functions_from_capability_implementation( - DummyCapabilityImplementation, FAKE_HIERARCHY_CONFIG, set() + status_type_adapter + ) = get_schema_and_functions_from_capability_implementations( + [dummy_cap], FAKE_HIERARCHY_CONFIG, set() ) + assert status_fn_capability is dummy_cap assert status_fn_name == 'get_status' - - assert status_fn_name in function_map assert status_fn_name not in schema['channels'] - assert status_type_adapter == function_map[status_fn_name].response_adapter - assert function_map[status_fn_name].request_adapter is None + + scoped_name = f'{status_fn_capability.capability_name}.{status_fn_name}' + assert scoped_name in function_map + assert status_type_adapter == function_map[scoped_name].response_adapter + assert function_map[scoped_name].request_adapter is None assert status_type_adapter.json_schema() == schema['components']['schemas']['DummyStatus'] def test_verify_attributes(): - _, function_map, _, _, _ = get_schema_and_functions_from_capability_implementation( - DummyCapabilityImplementation, - FAKE_HIERARCHY_CONFIG, - set(), + dummy_cap = DummyCapabilityImplementation() + _, function_map, _, _, _, _ = get_schema_and_functions_from_capability_implementations( + [dummy_cap], FAKE_HIERARCHY_CONFIG, set() ) # test defaults assert ( - getattr(function_map['verify_float_dict'].method, RESPONSE_DATA) + getattr(function_map['DummyCapability.verify_float_dict'].method, RESPONSE_DATA) == IntersectDataHandler.MESSAGE ) - assert getattr(function_map['verify_nested'].method, REQUEST_CONTENT) == IntersectMimeType.JSON - assert getattr(function_map['verify_nested'].method, RESPONSE_CONTENT) == IntersectMimeType.JSON - assert getattr(function_map['verify_nested'].method, STRICT_VALIDATION) is False + assert getattr(function_map['DummyCapability.verify_nested'].method, REQUEST_CONTENT) == IntersectMimeType.JSON + assert getattr(function_map['DummyCapability.verify_nested'].method, RESPONSE_CONTENT) == IntersectMimeType.JSON + assert getattr(function_map['DummyCapability.verify_nested'].method, STRICT_VALIDATION) is False # test non-defaults assert ( - getattr(function_map['verify_nested'].method, RESPONSE_DATA) == IntersectDataHandler.MINIO + getattr(function_map['DummyCapability.verify_nested'].method, RESPONSE_DATA) == IntersectDataHandler.MINIO ) - assert getattr(function_map['ip4_to_ip6'].method, RESPONSE_CONTENT) == IntersectMimeType.STRING - assert getattr(function_map['test_path'].method, REQUEST_CONTENT) == IntersectMimeType.STRING - assert getattr(function_map['calculate_weird_algorithm'].method, STRICT_VALIDATION) is True + assert getattr(function_map['DummyCapability.ip4_to_ip6'].method, RESPONSE_CONTENT) == IntersectMimeType.STRING + assert getattr(function_map['DummyCapability.test_path'].method, REQUEST_CONTENT) == IntersectMimeType.STRING + assert getattr(function_map['DummyCapability.calculate_weird_algorithm'].method, STRICT_VALIDATION) is True From cdf3912fdaa493f24549931fff4b57714580bf5c Mon Sep 17 00:00:00 2001 From: Lance Drane Date: Thu, 1 Aug 2024 16:39:57 -0400 Subject: [PATCH 02/10] svc2svc - fix a few typing errors, lint/format Signed-off-by: Lance Drane --- .pre-commit-config.yaml | 2 +- examples/1_hello_world/hello_service.py | 2 +- examples/1_hello_world_amqp/hello_service.py | 2 +- .../1_hello_world_events/hello_service.py | 2 +- examples/2_counting/counting_service.py | 2 +- .../2_counting_events/counting_service.py | 2 +- pyproject.toml | 2 +- .../_internal/messages/userspace.py | 15 +- src/intersect_sdk/_internal/schema.py | 39 ++-- src/intersect_sdk/capability/base.py | 11 +- .../client_callback_definitions.py | 7 +- src/intersect_sdk/service.py | 217 ++++++++++-------- tests/fixtures/example_schema.py | 2 +- .../integration/test_return_type_mismatch.py | 2 +- tests/unit/test_schema_valid.py | 41 ++-- 15 files changed, 203 insertions(+), 145 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 29138c9..1d0d688 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,7 +14,7 @@ repos: args: [ --fix ] - id: ruff-format - repo: https://github.com/pdm-project/pdm - rev: 2.11.2 + rev: 2.17.2 hooks: - id: pdm-export args: ['--without-hashes'] diff --git a/examples/1_hello_world/hello_service.py b/examples/1_hello_world/hello_service.py index 12d93db..e0be5aa 100644 --- a/examples/1_hello_world/hello_service.py +++ b/examples/1_hello_world/hello_service.py @@ -79,7 +79,7 @@ def say_hello_to_name(self, name: str) -> str: @intersect_message and @intersect_status, and that these functions are appropriately type-annotated. """ capability = HelloServiceCapabilityImplementation() - capability.capability_name = "HelloExample" + capability.capability_name = 'HelloExample' """ step three - create service from both the configuration and your own capability diff --git a/examples/1_hello_world_amqp/hello_service.py b/examples/1_hello_world_amqp/hello_service.py index 30e1ce8..192ee63 100644 --- a/examples/1_hello_world_amqp/hello_service.py +++ b/examples/1_hello_world_amqp/hello_service.py @@ -80,7 +80,7 @@ def say_hello_to_name(self, name: str) -> str: @intersect_message and @intersect_status, and that these functions are appropriately type-annotated. """ capability = HelloServiceCapabilityImplementation() - capability.capability_name = "HelloExample" + capability.capability_name = 'HelloExample' """ step three - create service from both the configuration and your own capability diff --git a/examples/1_hello_world_events/hello_service.py b/examples/1_hello_world_events/hello_service.py index 4591cbc..97b9cb1 100644 --- a/examples/1_hello_world_events/hello_service.py +++ b/examples/1_hello_world_events/hello_service.py @@ -83,7 +83,7 @@ def say_hello_to_name(self, name: str) -> str: @intersect_message and @intersect_status, and that these functions are appropriately type-annotated. """ capability = HelloServiceCapabilityImplementation() - capability.capability_name = "HelloExample" + capability.capability_name = 'HelloExample' """ step three - create service from both the configuration and your own capability diff --git a/examples/2_counting/counting_service.py b/examples/2_counting/counting_service.py index 1ba09c0..9a32176 100644 --- a/examples/2_counting/counting_service.py +++ b/examples/2_counting/counting_service.py @@ -191,7 +191,7 @@ def _run_count(self) -> None: **from_config_file, ) capability = CountingServiceCapabilityImplementation() - capability.capability_name = "CountingExample" + capability.capability_name = 'CountingExample' service = IntersectService([capability], config) logger.info('Starting counting_service, use Ctrl+C to exit.') default_intersect_lifecycle_loop( diff --git a/examples/2_counting_events/counting_service.py b/examples/2_counting_events/counting_service.py index d86933d..93974a9 100644 --- a/examples/2_counting_events/counting_service.py +++ b/examples/2_counting_events/counting_service.py @@ -83,7 +83,7 @@ def increment_counter_function(self) -> None: **from_config_file, ) capability = CountingServiceCapabilityImplementation() - capability.capability_name = "CountingExample" + capability.capability_name = 'CountingExample' service = IntersectService([capability], config) logger.info('Starting counting_service, use Ctrl+C to exit.') diff --git a/pyproject.toml b/pyproject.toml index db87f96..48d4431 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,7 +84,7 @@ isort = { known-first-party = ['src'] } pydocstyle = { convention = 'google'} flake8-quotes = {inline-quotes = 'single', multiline-quotes = 'double'} mccabe = { max-complexity = 20 } -pylint = { max-args = 10, max-branches = 20, max-returns = 10 } +pylint = { max-args = 10, max-branches = 20, max-returns = 10, max-statements = 75 } # pyflakes and the relevant pycodestyle rules are already configured extend-select = [ 'C90', # mccabe complexity diff --git a/src/intersect_sdk/_internal/messages/userspace.py b/src/intersect_sdk/_internal/messages/userspace.py index dd06e25..d838b86 100644 --- a/src/intersect_sdk/_internal/messages/userspace.py +++ b/src/intersect_sdk/_internal/messages/userspace.py @@ -9,6 +9,8 @@ from services they explicitly messaged. """ +from __future__ import annotations + import datetime import uuid from typing import Any, Union @@ -16,10 +18,13 @@ from pydantic import AwareDatetime, Field, TypeAdapter from typing_extensions import Annotated, TypedDict -from ...constants import SYSTEM_OF_SYSTEM_REGEX -from ...core_definitions import IntersectDataHandler, IntersectMimeType +from ...constants import SYSTEM_OF_SYSTEM_REGEX # noqa: TCH001 (this is runtime checked) +from ...core_definitions import ( # noqa: TCH001 (this is runtime checked) + IntersectDataHandler, + IntersectMimeType, +) from ...version import version_string -from ..data_plane.minio_utils import MinioPayload +from ..data_plane.minio_utils import MinioPayload # noqa: TCH001 (this is runtime checked) class UserspaceMessageHeader(TypedDict): @@ -115,7 +120,7 @@ class UserspaceMessage(TypedDict): the headers of the message """ - payload: Union[bytes, MinioPayload] # noqa: FA100 (Pydantic uses runtime annotations) + payload: Union[bytes, MinioPayload] # noqa: UP007 (Pydantic uses runtime annotations) """ main payload of the message. Needs to match the schema format, including the content type. @@ -141,7 +146,7 @@ def create_userspace_message( content_type: IntersectMimeType, data_handler: IntersectDataHandler, payload: Any, - message_id: uuid.UUID = None, + message_id: uuid.UUID | None = None, has_error: bool = False, ) -> UserspaceMessage: """Payloads depend on the data_handler and has_error.""" diff --git a/src/intersect_sdk/_internal/schema.py b/src/intersect_sdk/_internal/schema.py index 7abb6c4..daf235e 100644 --- a/src/intersect_sdk/_internal/schema.py +++ b/src/intersect_sdk/_internal/schema.py @@ -8,7 +8,6 @@ TYPE_CHECKING, Any, Callable, - List, Mapping, NamedTuple, get_origin, @@ -339,7 +338,7 @@ def _introspection_baseline( # parse functions for class_name, name, method, min_params in response_funcs: public_name = f'{cap_name}.{name}' - + # TODO - I'm placing this here for now because we'll eventually want to capture data plane and broker configs in the schema. # (It's possible we may want to separate the backing service schema from the application logic, but it's unlikely.) # At the moment, we're just validating that users can support their response_data_handler property. @@ -495,7 +494,7 @@ def _introspection_baseline( def get_schema_and_functions_from_capability_implementations( - capabilities: List[IntersectBaseCapabilityImplementation], + capabilities: list[IntersectBaseCapabilityImplementation], service_name: HierarchyConfig, excluded_data_handlers: set[IntersectDataHandler], ) -> tuple[ @@ -510,16 +509,16 @@ def get_schema_and_functions_from_capability_implementations( In-depth introspection is handled later on. """ - capability_type_docs : str = "" - status_function_cap : IntersectBaseCapabilityImplementation = None - status_function_name : str = None - status_function_schema : dict[str, Any] = None - status_function_adapter : TypeAdapter[Any] = None - schemas : dict[Any, Any] = dict() - channels : dict[str, dict[str, dict[str, Any]]] = dict() # endpoint schemas - function_map : dict[str, FunctionMetadata] = dict() # endpoint functionality - events : dict[str, Any] = dict() # event schemas - event_map : dict[str, EventMetadata] = dict() # event functionality + capability_type_docs: str = '' + status_function_cap: IntersectBaseCapabilityImplementation | None = None + status_function_name: str | None = None + status_function_schema: dict[str, Any] | None = None + status_function_adapter: TypeAdapter[Any] | None = None + schemas: dict[Any, Any] = {} + channels: dict[str, dict[str, dict[str, Any]]] = {} # endpoint schemas + function_map: dict[str, FunctionMetadata] = {} # endpoint functionality + events: dict[str, Any] = {} # event schemas + event_map: dict[str, EventMetadata] = {} # event functionality for capability in capabilities: capability_type: type[IntersectBaseCapabilityImplementation] = type(capability) if capability_type.__doc__: @@ -532,7 +531,7 @@ def get_schema_and_functions_from_capability_implementations( cap_events, cap_event_map, ) = _introspection_baseline(capability, excluded_data_handlers) - + if cap_status_fn_name and cap_status_schema and cap_status_type_adapter: status_function_cap = capability status_function_name = cap_status_fn_name @@ -544,7 +543,6 @@ def get_schema_and_functions_from_capability_implementations( function_map.update(cap_function_map) events.update(cap_events) event_map.update(cap_event_map) - asyncapi_spec = { 'asyncapi': ASYNCAPI_VERSION, @@ -576,7 +574,7 @@ def get_schema_and_functions_from_capability_implementations( }, } - if capability_type_docs != "": + if capability_type_docs != '': asyncapi_spec['info']['description'] = capability_type_docs # type: ignore[index] if status_function_schema: @@ -591,4 +589,11 @@ def get_schema_and_functions_from_capability_implementations( }, """ - return asyncapi_spec, function_map, event_map, status_function_cap, status_function_name, status_function_adapter + return ( + asyncapi_spec, + function_map, + event_map, + status_function_cap, + status_function_name, + status_function_adapter, + ) diff --git a/src/intersect_sdk/capability/base.py b/src/intersect_sdk/capability/base.py index e7b6716..a34973b 100644 --- a/src/intersect_sdk/capability/base.py +++ b/src/intersect_sdk/capability/base.py @@ -26,8 +26,7 @@ def __init__(self) -> None: NOTE: If you write your own constructor, you MUST call super.__init__() inside of it. The Service will throw an error if you don't. """ - - self._capability_name : str = "InvalidCapability" + self._capability_name: str = 'InvalidCapability' """ The advertised name for the capability, as opposed to the implementation class name """ @@ -49,14 +48,14 @@ def __init_subclass__(cls) -> None: ): msg = f"{cls.__name__}: Cannot override functions '_intersect_sdk_register_observer' or 'intersect_sdk_emit_event'" raise RuntimeError(msg) - + @property def capability_name(self) -> str: - """The advertised name for the capability provided by this implementation""" + """The advertised name for the capability provided by this implementation.""" return self._capability_name - + @capability_name.setter - def capability_name(self, cname : str) -> str: + def capability_name(self, cname: str) -> None: self._capability_name = cname @final diff --git a/src/intersect_sdk/client_callback_definitions.py b/src/intersect_sdk/client_callback_definitions.py index 5ffe2db..cac9188 100644 --- a/src/intersect_sdk/client_callback_definitions.py +++ b/src/intersect_sdk/client_callback_definitions.py @@ -1,6 +1,5 @@ """Data types used in regard to client callbacks. Only relevant for Client authors.""" -from dataclasses import dataclass from typing import Any, Callable, Dict, List, Optional, Union from pydantic import BaseModel, ConfigDict, Field @@ -10,8 +9,7 @@ from .core_definitions import IntersectDataHandler, IntersectMimeType -@dataclass -class IntersectClientMessageParams: +class IntersectClientMessageParams(BaseModel): """The user implementing the IntersectClient class will need to return this object in order to send a message to another Service.""" destination: Annotated[str, Field(pattern=SYSTEM_OF_SYSTEM_REGEX)] @@ -47,6 +45,9 @@ class IntersectClientMessageParams: default: IntersectDataHandler.MESSAGE """ + # pydantic config + model_config = ConfigDict(revalidate_instances='always') + @final class IntersectClientCallback(BaseModel): diff --git a/src/intersect_sdk/service.py b/src/intersect_sdk/service.py index 220edfe..1f4e1ff 100644 --- a/src/intersect_sdk/service.py +++ b/src/intersect_sdk/service.py @@ -19,10 +19,10 @@ from datetime import datetime, timezone from threading import Condition, Lock from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Callable, List, Tuple -from uuid import uuid1, uuid3, UUID +from typing import TYPE_CHECKING, Any, Callable +from uuid import UUID, uuid1, uuid3 -from pydantic import ValidationError +from pydantic import ConfigDict, ValidationError, validate_call from pydantic_core import PydanticSerializationError from typing_extensions import Self, final @@ -33,8 +33,8 @@ STRICT_VALIDATION, ) from ._internal.control_plane.control_plane_manager import ( + GENERIC_MESSAGE_SERIALIZER, ControlPlaneManager, - GENERIC_MESSAGE_SERIALIZER ) from ._internal.data_plane.data_plane_manager import DataPlaneManager from ._internal.exceptions import IntersectApplicationError, IntersectError @@ -52,7 +52,7 @@ from ._internal.utils import die from ._internal.version_resolver import resolve_user_version from .capability.base import IntersectBaseCapabilityImplementation -from .client_callback_definitions import IntersectClientMessageParams +from .client_callback_definitions import INTERSECT_JSON_VALUE, IntersectClientMessageParams from .config.service import IntersectServiceConfig from .core_definitions import IntersectDataHandler, IntersectMimeType from .version import version_string @@ -61,6 +61,9 @@ from ._internal.function_metadata import FunctionMetadata +RESPONSE_CALLBACK_TYPE = Callable[[INTERSECT_JSON_VALUE], None] + + @final class IntersectService(IntersectEventObserver): """This is the core gateway class for a user's application into INTERSECT. @@ -97,26 +100,30 @@ class IntersectService(IntersectEventObserver): No other functions or parameters are guaranteed to remain stable. """ - class ExternalRequest: - request_id : UUID - request_name : str - cv : Condition - processed : bool = False - error : str = None - request : Any = None - response : Any = None - response_fn : Callable = None - waiting : bool = False - - def __init__(self, req_id : UUID, req_name : str) -> None: + class _ExternalRequest: + """Class representative of an ongoing request to another service.""" + + def __init__( + self, + req_id: UUID, + req_name: str, + request: IntersectClientMessageParams, + response_handler: RESPONSE_CALLBACK_TYPE | None = None, + ) -> None: + """Create an external request.""" self.cv = Condition() self.request_id = req_id self.request_name = req_name - + self.processed: bool = False + self.error: str | None = None + self.request = request + self.response: INTERSECT_JSON_VALUE = None + self.response_fn = response_handler + self.waiting: bool = False def __init__( self, - capabilities: List[IntersectBaseCapabilityImplementation], + capabilities: list[IntersectBaseCapabilityImplementation], config: IntersectServiceConfig, ) -> None: """The constructor performs almost all validation checks necessary to function in the INTERSECT ecosystem, with the exception of checking connections/credentials to any backing services. @@ -125,7 +132,6 @@ def __init__( capabilities: Your list of capability implementation classes config: The IntersectConfig class """ - for cap in capabilities: if not isinstance(cap, IntersectBaseCapabilityImplementation): die( @@ -135,22 +141,22 @@ def __init__( die( f'{cap.__class__.__name__} needs to call "super().__init__()" in the constructor.' ) - + # we generally start observing and don't stop, doesn't really matter if we startup or shutdown cap._intersect_sdk_register_observer(self) # noqa: SLF001 (we don't want users calling or overriding it, but this is fine.) - + self.capabilities = capabilities - + # this is called here in case a user created the object using "IntersectServiceConfig.model_construct()" to skip validation config = IntersectServiceConfig.model_validate(config) - + ( schema, function_map, event_map, status_fn_capability, status_fn_name, - status_type_adapter + status_type_adapter, ) = get_schema_and_functions_from_capability_implementations( self.capabilities, service_name=config.hierarchy, @@ -160,7 +166,7 @@ def __init__( """ Stringified schema of the user's application. Gets sent in several status message requests. """ - + self._function_map = MappingProxyType(function_map) """ INTERNAL USE ONLY @@ -170,7 +176,7 @@ def __init__( You can get user-defined properties from the method via getattr(_function_map.method, KEY), the keys get set in the intersect_message decorator function (annotations.py). """ - + self._event_map = MappingProxyType(event_map) """ INTERNAL USE ONLY @@ -206,16 +212,20 @@ def __init__( self._status_memo = self._status_retrieval_fn() - self._external_request_thread = None + self._external_request_thread: StoppableThread | None = None self._external_requests_lock = Lock() - self._external_requests = dict() + self._external_requests: dict[str, IntersectService._ExternalRequest] = {} self._external_request_ctr = 0 - self._startup_messages : List[ Tuple[IntersectClientMessageParams, Callable] ] = list() + self._startup_messages: list[ + tuple[IntersectClientMessageParams, RESPONSE_CALLBACK_TYPE] + ] = [] self._resend_startup_messages = True self._sent_startup_messages = False - self._shutdown_messages : List[ Tuple[IntersectClientMessageParams, Callable] ] = list() + self._shutdown_messages: list[ + tuple[IntersectClientMessageParams, RESPONSE_CALLBACK_TYPE] + ] = [] self._data_plane_manager = DataPlaneManager(self._hierarchy, config.data_stores) # we PUBLISH messages on this channel @@ -226,23 +236,19 @@ def __init__( self._service_channel_name = f"{config.hierarchy.hierarchy_string('/')}/request" # we SUBSCRIBE to messages on this channel to receive responses self._client_channel_name = f"{config.hierarchy.hierarchy_string('/')}/response" - + self._control_plane_manager = ControlPlaneManager( control_configs=config.brokers, ) # our userspace queue should be able to survive shutdown self._control_plane_manager.add_subscription_channel( - self._service_channel_name, - {self._handle_service_message_raw}, - persist=True + self._service_channel_name, {self._handle_service_message_raw}, persist=True ) self._control_plane_manager.add_subscription_channel( - self._client_channel_name, - {self._handle_client_message_raw}, - persist=False + self._client_channel_name, {self._handle_client_message_raw}, persist=True ) - def _get_capability(self, target : str) -> Any | None: + def _get_capability(self, target: str) -> Any | None: for cap in self.capabilities: if cap.capability_name == target: return cap @@ -288,15 +294,15 @@ def startup(self) -> Self: logger.info('Sending startup messages') for tup in self._startup_messages: message, fn = tup - self.create_external_request(request=message, - response_handler=fn) - self.process_external_requests() + self.create_external_request(request=message, response_handler=fn) + self._process_external_requests() self._sent_startup_messages = True # Start the external request thread if it doesn't already exist if self._external_request_thread is None: self._external_request_thread = StoppableThread( - target=self._send_external_requests, name=f'IntersectService_{self._uuid}_ext_req_thread' + target=self._send_external_requests, + name=f'IntersectService_{self._uuid}_ext_req_thread', ) self._external_request_thread.start() @@ -328,9 +334,8 @@ def shutdown(self, reason: str | None = None) -> Self: logger.info('Sending shutdown messages') for tup in self._shutdown_messages: message, fn = tup - self.create_external_request(request=message, - response_handler=fn) - self.process_external_requests() + self.create_external_request(request=message, response_handler=fn) + self._process_external_requests() # Stop polling if self._status_thread is not None: @@ -437,53 +442,85 @@ def get_blocked_keys(self) -> set[str]: """ return self._function_keys.copy() - def add_startup_messages(self, messages : List[Tuple[IntersectClientMessageParams, Callable]]) -> None: + def add_startup_messages( + self, messages: list[tuple[IntersectClientMessageParams, RESPONSE_CALLBACK_TYPE]] + ) -> None: + """Add request messages to send out to various microservices when this service starts. + + Params: + - messages: list of tuples - first value in tuple contains the messages you want to send out on startup, + second value in tuple contains the callback function for handling the response from the other Service. + """ self._startup_messages.extend(messages) - def add_shutdown_messages(self, messages : List[Tuple[IntersectClientMessageParams, Callable]]) -> None: + def add_shutdown_messages( + self, messages: list[tuple[IntersectClientMessageParams, RESPONSE_CALLBACK_TYPE]] + ) -> None: + """Add request messages to send out to various microservices on shutdown. + + Params: + - messages: list of tuples - first value in tuple contains the messages you want to send out on startup, + second value in tuple contains the callback function for handling the response from the other Service. + """ self._shutdown_messages.extend(messages) - def _new_external_request(self) -> IntersectService.ExternalRequest: + @validate_call(config=ConfigDict(revalidate_instances='always')) + def create_external_request( + self, + request: IntersectClientMessageParams, + response_handler: RESPONSE_CALLBACK_TYPE | None = None, + ) -> UUID: + """Create an external request structure with a Condition we can wait on. + + Params: + - request: the request we want to send out, encapsulated as an IntersectClientMessageParams object + - response_handler: optional callback for how we want to handle the response from this request. + + Returns: + - generated RequestID associated with your request + + Raises: + - pydantic.ValidationError - if the request parameter isn't valid + """ self._external_request_ctr += 1 request_name = f'ext-req-{self._external_request_ctr}' request_uuid = uuid3(self._uuid, request_name) - req = IntersectService.ExternalRequest(req_id=request_uuid, - req_name=request_name) + extreq = IntersectService._ExternalRequest( + req_id=request_uuid, + req_name=request_name, + request=request, + response_handler=response_handler, + ) self._external_requests_lock.acquire_lock(blocking=True) - self._external_requests[str(request_uuid)] = req + self._external_requests[str(request_uuid)] = extreq self._external_requests_lock.release_lock() - return req - - def _delete_external_request(self, req_id : UUID) -> None: + return request_uuid + + def _delete_external_request(self, req_id: UUID) -> None: req_id_str = str(req_id) if req_id_str in self._external_requests: - req : IntersectService.ExternalRequest = self._external_requests.pop(req_id_str) + self._external_requests_lock.acquire_lock(blocking=True) + req: IntersectService._ExternalRequest = self._external_requests.pop(req_id_str) del req + self._external_requests_lock.release_lock() - def _get_external_request(self, req_id : UUID) -> IntersectService.ExternalRequest: + def _get_external_request(self, req_id: UUID) -> IntersectService._ExternalRequest | None: req_id_str = str(req_id) if req_id_str in self._external_requests: - req : IntersectService.ExternalRequest = self._external_requests[req_id_str] + req: IntersectService._ExternalRequest = self._external_requests[req_id_str] return req return None - def create_external_request(self, - request: IntersectClientMessageParams, - response_handler : Callable = None) -> UUID: - # create an external request structure with a Condition we can wait on - extreq : IntersectService.ExternalRequest = self._new_external_request() - extreq.request = request - extreq.response_fn = response_handler - return extreq.request_id - - def process_external_requests(self) -> None: + def _process_external_requests(self) -> None: self._external_requests_lock.acquire_lock(blocking=True) for extreq in self._external_requests.values(): if not extreq.processed: self._process_external_request(extreq) self._external_requests_lock.release_lock() - def _process_external_request(self, extreq: IntersectService.ExternalRequest) -> None: + def _process_external_request(self, extreq: IntersectService._ExternalRequest) -> None: + if extreq.request is None: + return response = None cleanup_req = False @@ -498,7 +535,7 @@ def _process_external_request(self, extreq: IntersectService.ExternalRequest) -> # external request when this function is called while # handling an incoming request, so we are just ignoring # any wait timeouts below. - + # wait on the response condition and get the response extreq.waiting = True if extreq.cv.wait(timeout=1.0): @@ -506,7 +543,9 @@ def _process_external_request(self, extreq: IntersectService.ExternalRequest) -> response = extreq.response else: error_msg = extreq.error - logger.warning(f'External service request encountered an error: {error_msg}') + logger.warning( + f'External service request encountered an error: {error_msg}' + ) cleanup_req = True else: logger.debug('Request wait timed-out!') @@ -515,13 +554,12 @@ def _process_external_request(self, extreq: IntersectService.ExternalRequest) -> logger.warning('Failed to send request!') # process the response - if response is not None: - if extreq.response_fn is not None: - extreq.response_fn(response) + if extreq.response_fn is not None and extreq.error is None: + extreq.response_fn(response) if cleanup_req: - self._delete_external_request(extreq.request_name) - + self._delete_external_request(extreq.request_id) + def _handle_service_message_raw(self, raw: bytes) -> None: """Main broker callback function. @@ -579,7 +617,7 @@ def _handle_service_message(self, message: UserspaceMessage) -> UserspaceMessage err_msg = f"Function '{operation}' is currently not available for use." logger.error(err_msg) return self._make_error_message(err_msg, message) - + operation_capability, operation_method = operation.split('.') target_capability = self._get_capability(operation_capability) if target_capability is None: @@ -597,7 +635,9 @@ def _handle_service_message(self, message: UserspaceMessage) -> UserspaceMessage try: # FOUR: CALL USER FUNCTION AND GET MESSAGE - response = self._call_user_function(target_capability, operation_method, operation_meta, request_params) + response = self._call_user_function( + target_capability, operation_method, operation_meta, request_params + ) # FIVE: SEND DATA TO APPROPRIATE DATA STORE response_data_handler = getattr(operation_meta.method, RESPONSE_DATA) response_content_type = getattr(operation_meta.method, RESPONSE_CONTENT) @@ -624,9 +664,9 @@ def _handle_service_message(self, message: UserspaceMessage) -> UserspaceMessage data_handler=response_data_handler, operation_id=message['operationId'], payload=response_payload, - message_id=message['messageId'] # associate response with request + message_id=message['messageId'], # associate response with request ) - + def _handle_client_message_raw(self, raw: bytes) -> None: """Broker callback, deserialize and validate a userspace message from a broker.""" try: @@ -640,10 +680,9 @@ def _handle_client_message_raw(self, raw: bytes) -> None: def _handle_client_message(self, message: UserspaceMessage) -> None: """Handle a deserialized userspace message.""" - extreq = self._get_external_request(message['messageId']) if extreq is not None: - error_msg : str = None + error_msg: str | None = None try: msg_payload = GENERIC_MESSAGE_SERIALIZER.validate_json( self._data_plane_manager.incoming_message_data_handler(message) @@ -652,7 +691,7 @@ def _handle_client_message(self, message: UserspaceMessage) -> None: error_msg = f'Service sent back invalid response:\n{e}' logger.warning(error_msg) except IntersectError: - error_msg = f'INTERNAL ERROR: failed to get message payload from data handler' + error_msg = 'INTERNAL ERROR: failed to get message payload from data handler' logger.error(error_msg) with extreq.cv: @@ -666,7 +705,7 @@ def _handle_client_message(self, message: UserspaceMessage) -> None: error_msg = f'No external request found for message:\n{message}' logger.warning(error_msg) - def _send_client_message(self, request_id : UUID, params: IntersectClientMessageParams) -> bool: + def _send_client_message(self, request_id: UUID, params: IntersectClientMessageParams) -> bool: """Send a userspace message.""" # ONE: VALIDATE AND SERIALIZE FUNCTION RESULTS try: @@ -688,16 +727,15 @@ def _send_client_message(self, request_id : UUID, params: IntersectClientMessage msg = create_userspace_message( source=self._hierarchy.hierarchy_string('.'), destination=params.destination, - service_version=self._version, content_type=params.content_type, data_handler=params.data_handler, operation_id=params.operation, payload=request_payload, - message_id=request_id + message_id=request_id, ) logger.debug(f'Sending client message:\n{msg}') request_channel = f"{params.destination.replace('.', '/')}/request" - self._control_plane_manager.publish_message(request_channel, msg) + self._control_plane_manager.publish_message(request_channel, msg, persist=True) return True def _call_user_function( @@ -844,7 +882,7 @@ def _make_error_message( data_handler=IntersectDataHandler.MESSAGE, operation_id=original_message['operationId'], payload=error_string, - message_id=original_message['messageId'], # associate error reply with original + message_id=original_message['messageId'], # associate error reply with original has_error=True, ) @@ -903,6 +941,5 @@ def _send_external_requests(self) -> None: if self._external_request_thread: self._external_request_thread.wait(10.0) while not self._external_request_thread.stopped(): - self.process_external_requests() + self._process_external_requests() self._external_request_thread.wait(0.5) - diff --git a/tests/fixtures/example_schema.py b/tests/fixtures/example_schema.py index 26c2f92..b36edd2 100644 --- a/tests/fixtures/example_schema.py +++ b/tests/fixtures/example_schema.py @@ -235,7 +235,7 @@ def __init__(self) -> None: which handles talking to the various INTERSECT-related backing services. """ super().__init__() - self.capability_name = "DummyCapability" + self.capability_name = 'DummyCapability' self._status_example = DummyStatus( functions_called=0, last_function_called='', diff --git a/tests/integration/test_return_type_mismatch.py b/tests/integration/test_return_type_mismatch.py index e4a468b..c5c7954 100644 --- a/tests/integration/test_return_type_mismatch.py +++ b/tests/integration/test_return_type_mismatch.py @@ -45,7 +45,7 @@ def wrong_return_annotation(self, param: int) -> int: def make_intersect_service() -> IntersectService: capability = ReturnTypeMismatchCapabilityImplementation() - capability.capability_name = "ReturnTypeMismatchCapability" + capability.capability_name = 'ReturnTypeMismatchCapability' return IntersectService( [capability], IntersectServiceConfig( diff --git a/tests/unit/test_schema_valid.py b/tests/unit/test_schema_valid.py index aa238ea..2d10100 100644 --- a/tests/unit/test_schema_valid.py +++ b/tests/unit/test_schema_valid.py @@ -38,15 +38,10 @@ def test_schema_comparison(): def test_verify_status_fn(): dummy_cap = DummyCapabilityImplementation() - ( - schema, - function_map, - _, - status_fn_capability, - status_fn_name, - status_type_adapter - ) = get_schema_and_functions_from_capability_implementations( - [dummy_cap], FAKE_HIERARCHY_CONFIG, set() + (schema, function_map, _, status_fn_capability, status_fn_name, status_type_adapter) = ( + get_schema_and_functions_from_capability_implementations( + [dummy_cap], FAKE_HIERARCHY_CONFIG, set() + ) ) assert status_fn_capability is dummy_cap assert status_fn_name == 'get_status' @@ -69,14 +64,30 @@ def test_verify_attributes(): getattr(function_map['DummyCapability.verify_float_dict'].method, RESPONSE_DATA) == IntersectDataHandler.MESSAGE ) - assert getattr(function_map['DummyCapability.verify_nested'].method, REQUEST_CONTENT) == IntersectMimeType.JSON - assert getattr(function_map['DummyCapability.verify_nested'].method, RESPONSE_CONTENT) == IntersectMimeType.JSON + assert ( + getattr(function_map['DummyCapability.verify_nested'].method, REQUEST_CONTENT) + == IntersectMimeType.JSON + ) + assert ( + getattr(function_map['DummyCapability.verify_nested'].method, RESPONSE_CONTENT) + == IntersectMimeType.JSON + ) assert getattr(function_map['DummyCapability.verify_nested'].method, STRICT_VALIDATION) is False # test non-defaults assert ( - getattr(function_map['DummyCapability.verify_nested'].method, RESPONSE_DATA) == IntersectDataHandler.MINIO + getattr(function_map['DummyCapability.verify_nested'].method, RESPONSE_DATA) + == IntersectDataHandler.MINIO + ) + assert ( + getattr(function_map['DummyCapability.ip4_to_ip6'].method, RESPONSE_CONTENT) + == IntersectMimeType.STRING + ) + assert ( + getattr(function_map['DummyCapability.test_path'].method, REQUEST_CONTENT) + == IntersectMimeType.STRING + ) + assert ( + getattr(function_map['DummyCapability.calculate_weird_algorithm'].method, STRICT_VALIDATION) + is True ) - assert getattr(function_map['DummyCapability.ip4_to_ip6'].method, RESPONSE_CONTENT) == IntersectMimeType.STRING - assert getattr(function_map['DummyCapability.test_path'].method, REQUEST_CONTENT) == IntersectMimeType.STRING - assert getattr(function_map['DummyCapability.calculate_weird_algorithm'].method, STRICT_VALIDATION) is True From 2478e575bc976428a5d8ab2454ab28fb76b6feba Mon Sep 17 00:00:00 2001 From: Lance Drane Date: Thu, 1 Aug 2024 17:26:17 -0400 Subject: [PATCH 03/10] validate IntersectClientMessageParams in Service earlier Signed-off-by: Lance Drane --- src/intersect_sdk/service.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/src/intersect_sdk/service.py b/src/intersect_sdk/service.py index 1f4e1ff..e8ed843 100644 --- a/src/intersect_sdk/service.py +++ b/src/intersect_sdk/service.py @@ -91,6 +91,7 @@ class IntersectService(IntersectEventObserver): - shutdown() - add_shutdown_messages() - is_connected() + - considered_unrecoverable() - forbid_keys() - allow_keys() - allow_all_functions() @@ -218,13 +219,13 @@ def __init__( self._external_request_ctr = 0 self._startup_messages: list[ - tuple[IntersectClientMessageParams, RESPONSE_CALLBACK_TYPE] + tuple[IntersectClientMessageParams, RESPONSE_CALLBACK_TYPE | None] ] = [] self._resend_startup_messages = True self._sent_startup_messages = False self._shutdown_messages: list[ - tuple[IntersectClientMessageParams, RESPONSE_CALLBACK_TYPE] + tuple[IntersectClientMessageParams, RESPONSE_CALLBACK_TYPE | None] ] = [] self._data_plane_manager = DataPlaneManager(self._hierarchy, config.data_stores) @@ -443,7 +444,7 @@ def get_blocked_keys(self) -> set[str]: return self._function_keys.copy() def add_startup_messages( - self, messages: list[tuple[IntersectClientMessageParams, RESPONSE_CALLBACK_TYPE]] + self, messages: list[tuple[IntersectClientMessageParams, RESPONSE_CALLBACK_TYPE | None]] ) -> None: """Add request messages to send out to various microservices when this service starts. @@ -454,7 +455,7 @@ def add_startup_messages( self._startup_messages.extend(messages) def add_shutdown_messages( - self, messages: list[tuple[IntersectClientMessageParams, RESPONSE_CALLBACK_TYPE]] + self, messages: list[tuple[IntersectClientMessageParams, RESPONSE_CALLBACK_TYPE | None]] ) -> None: """Add request messages to send out to various microservices on shutdown. @@ -519,8 +520,6 @@ def _process_external_requests(self) -> None: self._external_requests_lock.release_lock() def _process_external_request(self, extreq: IntersectService._ExternalRequest) -> None: - if extreq.request is None: - return response = None cleanup_req = False @@ -707,12 +706,7 @@ def _handle_client_message(self, message: UserspaceMessage) -> None: def _send_client_message(self, request_id: UUID, params: IntersectClientMessageParams) -> bool: """Send a userspace message.""" - # ONE: VALIDATE AND SERIALIZE FUNCTION RESULTS - try: - params = IntersectClientMessageParams.model_validate(params) - except ValidationError as e: - logger.error(f'Invalid message parameters:\n{e}') - return False + # "params" should already be validated at this stage. request = GENERIC_MESSAGE_SERIALIZER.dump_json(params.payload, warnings=False) # TWO: SEND DATA TO APPROPRIATE DATA STORE From ef97298c21ce0713308e779107ad56531c1d49db Mon Sep 17 00:00:00 2001 From: Lance Drane Date: Fri, 2 Aug 2024 16:59:14 -0400 Subject: [PATCH 04/10] add functionality for capability to send out messages, rename 'IntersectClientMessageParams' to 'DirectMessageParams' Signed-off-by: Lance Drane --- examples/1_hello_world/hello_client.py | 4 +- examples/1_hello_world_amqp/hello_client.py | 4 +- examples/1_hello_world_events/hello_client.py | 4 +- examples/2_counting/counting_client.py | 16 ++--- src/intersect_sdk/__init__.py | 12 +++- src/intersect_sdk/_internal/interfaces.py | 31 +++++++- .../_internal/messages/userspace.py | 7 ++ src/intersect_sdk/capability/base.py | 42 ++++++++++- src/intersect_sdk/client.py | 4 +- .../client_callback_definitions.py | 51 ++------------ src/intersect_sdk/service.py | 43 +++++++----- .../service_callback_definitions.py | 16 +++++ .../shared_callback_definitions.py | 67 ++++++++++++++++++ .../test_base_capability_implementation.py | 70 ++++++++++++++++++- 14 files changed, 287 insertions(+), 84 deletions(-) create mode 100644 src/intersect_sdk/service_callback_definitions.py create mode 100644 src/intersect_sdk/shared_callback_definitions.py diff --git a/examples/1_hello_world/hello_client.py b/examples/1_hello_world/hello_client.py index 1c7f589..3a8a29c 100644 --- a/examples/1_hello_world/hello_client.py +++ b/examples/1_hello_world/hello_client.py @@ -2,10 +2,10 @@ from intersect_sdk import ( INTERSECT_JSON_VALUE, + DirectMessageParams, IntersectClient, IntersectClientCallback, IntersectClientConfig, - IntersectClientMessageParams, default_intersect_lifecycle_loop, ) @@ -74,7 +74,7 @@ def simple_client_callback( you'll get a message back. """ initial_messages = [ - IntersectClientMessageParams( + DirectMessageParams( destination='hello-organization.hello-facility.hello-system.hello-subsystem.hello-service', operation='HelloExample.say_hello_to_name', payload='hello_client', diff --git a/examples/1_hello_world_amqp/hello_client.py b/examples/1_hello_world_amqp/hello_client.py index 7ce87f4..a0063f4 100644 --- a/examples/1_hello_world_amqp/hello_client.py +++ b/examples/1_hello_world_amqp/hello_client.py @@ -2,10 +2,10 @@ from intersect_sdk import ( INTERSECT_JSON_VALUE, + DirectMessageParams, IntersectClient, IntersectClientCallback, IntersectClientConfig, - IntersectClientMessageParams, default_intersect_lifecycle_loop, ) @@ -76,7 +76,7 @@ def simple_client_callback( you'll get a message back. """ initial_messages = [ - IntersectClientMessageParams( + DirectMessageParams( destination='hello-organization.hello-facility.hello-system.hello-subsystem.hello-service', operation='HelloExample.say_hello_to_name', payload='hello_client', diff --git a/examples/1_hello_world_events/hello_client.py b/examples/1_hello_world_events/hello_client.py index 86c44ff..15e47af 100644 --- a/examples/1_hello_world_events/hello_client.py +++ b/examples/1_hello_world_events/hello_client.py @@ -2,10 +2,10 @@ from intersect_sdk import ( INTERSECT_JSON_VALUE, + DirectMessageParams, IntersectClient, IntersectClientCallback, IntersectClientConfig, - IntersectClientMessageParams, default_intersect_lifecycle_loop, ) @@ -95,7 +95,7 @@ def simple_event_callback( you'll get a message back. """ initial_messages = [ - IntersectClientMessageParams( + DirectMessageParams( destination='hello-organization.hello-facility.hello-system.hello-subsystem.hello-service', operation='HelloExample.say_hello_to_name', payload='hello_client', diff --git a/examples/2_counting/counting_client.py b/examples/2_counting/counting_client.py index 0bf758b..7868fe3 100644 --- a/examples/2_counting/counting_client.py +++ b/examples/2_counting/counting_client.py @@ -4,10 +4,10 @@ from intersect_sdk import ( INTERSECT_JSON_VALUE, + DirectMessageParams, IntersectClient, IntersectClientCallback, IntersectClientConfig, - IntersectClientMessageParams, default_intersect_lifecycle_loop, ) @@ -38,7 +38,7 @@ def __init__(self) -> None: self.message_stack = [ # wait 5 seconds before stopping the counter. "Count" in response will be approx. 6 ( - IntersectClientMessageParams( + DirectMessageParams( destination='counting-organization.counting-facility.counting-system.counting-subsystem.counting-service', operation='CountingExample.stop_count', payload=None, @@ -47,7 +47,7 @@ def __init__(self) -> None: ), # start the counter up again - it will not be 0 at this point! "Count" in response will be approx. 7 ( - IntersectClientMessageParams( + DirectMessageParams( destination='counting-organization.counting-facility.counting-system.counting-subsystem.counting-service', operation='CountingExample.start_count', payload=None, @@ -56,7 +56,7 @@ def __init__(self) -> None: ), # reset the counter, but have it immediately start running again. "Count" in response will be approx. 10 ( - IntersectClientMessageParams( + DirectMessageParams( destination='counting-organization.counting-facility.counting-system.counting-subsystem.counting-service', operation='CountingExample.reset_count', payload=True, @@ -65,7 +65,7 @@ def __init__(self) -> None: ), # reset the counter, but don't have it run again. "Count" in response will be approx. 6 ( - IntersectClientMessageParams( + DirectMessageParams( destination='counting-organization.counting-facility.counting-system.counting-subsystem.counting-service', operation='CountingExample.reset_count', payload=False, @@ -74,7 +74,7 @@ def __init__(self) -> None: ), # start the counter back up. "Count" in response will be approx. 1 ( - IntersectClientMessageParams( + DirectMessageParams( destination='counting-organization.counting-facility.counting-system.counting-subsystem.counting-service', operation='CountingExample.start_count', payload=None, @@ -83,7 +83,7 @@ def __init__(self) -> None: ), # finally, stop the counter one last time. "Count" in response will be approx. 4 ( - IntersectClientMessageParams( + DirectMessageParams( destination='counting-organization.counting-facility.counting-system.counting-subsystem.counting-service', operation='CountingExample.stop_count', payload=None, @@ -158,7 +158,7 @@ def client_callback( # The counter will start after the initial message. # If the service is already active and counting, this may do nothing. initial_messages = [ - IntersectClientMessageParams( + DirectMessageParams( destination='counting-organization.counting-facility.counting-system.counting-subsystem.counting-service', operation='CountingExample.start_count', payload=None, diff --git a/src/intersect_sdk/__init__.py b/src/intersect_sdk/__init__.py index fd43448..52d5f4a 100644 --- a/src/intersect_sdk/__init__.py +++ b/src/intersect_sdk/__init__.py @@ -13,9 +13,7 @@ from .client_callback_definitions import ( INTERSECT_CLIENT_EVENT_CALLBACK_TYPE, INTERSECT_CLIENT_RESPONSE_CALLBACK_TYPE, - INTERSECT_JSON_VALUE, IntersectClientCallback, - IntersectClientMessageParams, ) from .config.client import IntersectClientConfig from .config.service import IntersectServiceConfig @@ -28,12 +26,19 @@ from .core_definitions import IntersectDataHandler, IntersectMimeType from .schema import get_schema_from_capability_implementation from .service import IntersectService +from .service_callback_definitions import ( + INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE, +) from .service_definitions import ( IntersectEventDefinition, intersect_event, intersect_message, intersect_status, ) +from .shared_callback_definitions import ( + INTERSECT_JSON_VALUE, + DirectMessageParams, +) from .version import __version__, version_info, version_string __all__ = [ @@ -47,10 +52,11 @@ 'IntersectService', 'IntersectClient', 'IntersectClientCallback', - 'IntersectClientMessageParams', + 'DirectMessageParams', 'INTERSECT_CLIENT_RESPONSE_CALLBACK_TYPE', 'INTERSECT_CLIENT_EVENT_CALLBACK_TYPE', 'INTERSECT_JSON_VALUE', + 'INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE', 'IntersectBaseCapabilityImplementation', 'default_intersect_lifecycle_loop', 'IntersectClientConfig', diff --git a/src/intersect_sdk/_internal/interfaces.py b/src/intersect_sdk/_internal/interfaces.py index b4308b0..197cf3f 100644 --- a/src/intersect_sdk/_internal/interfaces.py +++ b/src/intersect_sdk/_internal/interfaces.py @@ -1,5 +1,17 @@ +from __future__ import annotations + from abc import ABC, abstractmethod -from typing import Any +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from uuid import UUID + + from ..service_callback_definitions import ( + INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE, + ) + from ..shared_callback_definitions import ( + DirectMessageParams, + ) class IntersectEventObserver(ABC): @@ -18,3 +30,20 @@ def _on_observe_event(self, event_name: str, event_value: Any, operation: str) - operation: The source of the event (generally the function name, not directly invoked by application devs) """ ... + + @abstractmethod + def create_external_request( + self, + request: DirectMessageParams, + response_handler: INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE | None = None, + ) -> UUID: + """Observed entity (capabilitiy) tells observer (i.e. service) to send an external request. + + Params: + - request: the request we want to send out, encapsulated as an IntersectClientMessageParams object + - response_handler: optional callback for how we want to handle the response from this request. + + Returns: + - generated RequestID associated with your request + """ + ... diff --git a/src/intersect_sdk/_internal/messages/userspace.py b/src/intersect_sdk/_internal/messages/userspace.py index d838b86..22bc75e 100644 --- a/src/intersect_sdk/_internal/messages/userspace.py +++ b/src/intersect_sdk/_internal/messages/userspace.py @@ -2,6 +2,13 @@ This module is internal-facing and should not be used directly by users. +Services have two associated channels which handle userspace messages: their request channel +and their response channel. Services always CONSUME messages from these channels, but never PRODUCE messages +on these channels. (A message is always sent in the receiver's namespace). + +The response channel is how the service handles external requests, the request channel is used when this service itself +needs to make external requests through INTERSECT. + Services should ALWAYS be CONSUMING from their userspace channel. They should NEVER be PRODUCING messages on their userspace channel. diff --git a/src/intersect_sdk/capability/base.py b/src/intersect_sdk/capability/base.py index a34973b..bee048f 100644 --- a/src/intersect_sdk/capability/base.py +++ b/src/intersect_sdk/capability/base.py @@ -11,7 +11,15 @@ from .._internal.logger import logger if TYPE_CHECKING: + from uuid import UUID + from .._internal.interfaces import IntersectEventObserver + from ..service_callback_definitions import ( + INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE, + ) + from ..shared_callback_definitions import ( + DirectMessageParams, + ) class IntersectBaseCapabilityImplementation: @@ -39,14 +47,20 @@ def __init__(self) -> None: """ def __init_subclass__(cls) -> None: - """This prevents users from overriding a few key functions.""" + """This prevents users from overriding a few key functions. + + General rule of thumb is that any function which starts with "intersect_sdk_" is a protected namespace for defining + the INTERSECT-SDK public API between a capability and its observers. + """ if ( cls._intersect_sdk_register_observer is not IntersectBaseCapabilityImplementation._intersect_sdk_register_observer or cls.intersect_sdk_emit_event is not IntersectBaseCapabilityImplementation.intersect_sdk_emit_event + or cls.intersect_sdk_call_service + is not IntersectBaseCapabilityImplementation.intersect_sdk_call_service ): - msg = f"{cls.__name__}: Cannot override functions '_intersect_sdk_register_observer' or 'intersect_sdk_emit_event'" + msg = f"{cls.__name__}: Attempted to override a reserved INTERSECT-SDK function (don't start your function names with '_intersect_sdk_' or 'intersect_sdk_')" raise RuntimeError(msg) @property @@ -113,3 +127,27 @@ def intersect_sdk_emit_event(self, event_name: str, event_value: Any) -> None: return for observer in self.__intersect_sdk_observers__: observer._on_observe_event(event_name, event_value, annotated_operation) # noqa: SLF001 (private for application devs, NOT for base implementation) + + @final + def intersect_sdk_call_service( + self, + request: DirectMessageParams, + response_handler: INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE | None = None, + ) -> list[UUID]: + """Create an external request that we'll send to a different Service. + + Params: + - request: the request we want to send out, encapsulated as an IntersectClientMessageParams object + - response_handler: optional callback for how we want to handle the response from this request. + + Returns: + - list of generated RequestIDs associated with your request. Note that for almost all use cases, + this list will have only one associated RequestID. + + Raises: + - pydantic.ValidationError - if the request parameter isn't valid + """ + return [ + observer.create_external_request(request, response_handler) + for observer in self.__intersect_sdk_observers__ + ] diff --git a/src/intersect_sdk/client.py b/src/intersect_sdk/client.py index 4c53cfb..cc5fdb2 100644 --- a/src/intersect_sdk/client.py +++ b/src/intersect_sdk/client.py @@ -40,7 +40,6 @@ from ._internal.version_resolver import resolve_user_version from .client_callback_definitions import ( IntersectClientCallback, - IntersectClientMessageParams, ) from .config.client import IntersectClientConfig from .config.shared import HierarchyConfig @@ -50,6 +49,7 @@ INTERSECT_CLIENT_EVENT_CALLBACK_TYPE, INTERSECT_CLIENT_RESPONSE_CALLBACK_TYPE, ) + from .shared_callback_definitions import DirectMessageParams @final @@ -425,7 +425,7 @@ def _handle_client_callback(self, user_value: IntersectClientCallback | None) -> for message in validated_result.messages_to_send: self._send_userspace_message(message) - def _send_userspace_message(self, params: IntersectClientMessageParams) -> None: + def _send_userspace_message(self, params: DirectMessageParams) -> None: """Send a userspace message, be it an initial message from the user or from the user's callback function.""" # ONE: SERIALIZE FUNCTION RESULTS # (function input should already be validated at this point) diff --git a/src/intersect_sdk/client_callback_definitions.py b/src/intersect_sdk/client_callback_definitions.py index cac9188..453d7d4 100644 --- a/src/intersect_sdk/client_callback_definitions.py +++ b/src/intersect_sdk/client_callback_definitions.py @@ -1,52 +1,15 @@ -"""Data types used in regard to client callbacks. Only relevant for Client authors.""" +"""Data types used in regard to client callbacks. Only relevant for Client authors. -from typing import Any, Callable, Dict, List, Optional, Union +See shared_callback_definitions for additional typings which are also shared by service authors. +""" + +from typing import Callable, Dict, List, Optional, Union from pydantic import BaseModel, ConfigDict, Field from typing_extensions import Annotated, TypeAlias, final from .constants import SYSTEM_OF_SYSTEM_REGEX -from .core_definitions import IntersectDataHandler, IntersectMimeType - - -class IntersectClientMessageParams(BaseModel): - """The user implementing the IntersectClient class will need to return this object in order to send a message to another Service.""" - - destination: Annotated[str, Field(pattern=SYSTEM_OF_SYSTEM_REGEX)] - """ - The destination string. You'll need to know the system-of-system representation of the Service. - - Note that this should match what you would see in the schema. - """ - - operation: str - """ - The name of the operation you want to call from the Service - this should be represented as it is in the Service's schema. - """ - - payload: Any - """ - The raw Python object you want to have serialized as the payload. - - If you want to just use the service's default value for a request (assuming it has a default value for a request), you may set this as None. - """ - - content_type: IntersectMimeType = IntersectMimeType.JSON - """ - The IntersectMimeType of your message. You'll want this to match with the ContentType of the function from the schema. - - default: IntersectMimeType.JSON - """ - - data_handler: IntersectDataHandler = IntersectDataHandler.MESSAGE - """ - The IntersectDataHandler you want to use (most people can just use IntersectDataHandler.MESSAGE here, unless your data is very large) - - default: IntersectDataHandler.MESSAGE - """ - - # pydantic config - model_config = ConfigDict(revalidate_instances='always') +from .shared_callback_definitions import DirectMessageParams @final @@ -56,7 +19,7 @@ class IntersectClientCallback(BaseModel): If you do not return a value of this type (or None), this will be treated as an Exception and will break the pub-sub loop. """ - messages_to_send: List[IntersectClientMessageParams] = [] # noqa: FA100 (runtime annotation) + messages_to_send: List[DirectMessageParams] = [] # noqa: FA100 (runtime annotation) """ Messages to send as a result of an event or a response from a Service. """ diff --git a/src/intersect_sdk/service.py b/src/intersect_sdk/service.py index e8ed843..d9ff452 100644 --- a/src/intersect_sdk/service.py +++ b/src/intersect_sdk/service.py @@ -19,7 +19,7 @@ from datetime import datetime, timezone from threading import Condition, Lock from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Callable +from typing import TYPE_CHECKING, Any, Callable, Union from uuid import UUID, uuid1, uuid3 from pydantic import ConfigDict, ValidationError, validate_call @@ -52,18 +52,21 @@ from ._internal.utils import die from ._internal.version_resolver import resolve_user_version from .capability.base import IntersectBaseCapabilityImplementation -from .client_callback_definitions import INTERSECT_JSON_VALUE, IntersectClientMessageParams from .config.service import IntersectServiceConfig from .core_definitions import IntersectDataHandler, IntersectMimeType +from .service_callback_definitions import ( + INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE, # noqa: TCH001 (runtime-checked annotation) +) +from .shared_callback_definitions import ( + INTERSECT_JSON_VALUE, # noqa: TCH001 (runtime-checked annotation) + DirectMessageParams, # noqa: TCH001 (runtime-checked annotation) +) from .version import version_string if TYPE_CHECKING: from ._internal.function_metadata import FunctionMetadata -RESPONSE_CALLBACK_TYPE = Callable[[INTERSECT_JSON_VALUE], None] - - @final class IntersectService(IntersectEventObserver): """This is the core gateway class for a user's application into INTERSECT. @@ -108,8 +111,8 @@ def __init__( self, req_id: UUID, req_name: str, - request: IntersectClientMessageParams, - response_handler: RESPONSE_CALLBACK_TYPE | None = None, + request: DirectMessageParams, + response_handler: INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE | None = None, ) -> None: """Create an external request.""" self.cv = Condition() @@ -119,6 +122,7 @@ def __init__( self.error: str | None = None self.request = request self.response: INTERSECT_JSON_VALUE = None + self.got_valid_response: bool = False self.response_fn = response_handler self.waiting: bool = False @@ -219,13 +223,13 @@ def __init__( self._external_request_ctr = 0 self._startup_messages: list[ - tuple[IntersectClientMessageParams, RESPONSE_CALLBACK_TYPE | None] + tuple[DirectMessageParams, INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE | None] ] = [] self._resend_startup_messages = True self._sent_startup_messages = False self._shutdown_messages: list[ - tuple[IntersectClientMessageParams, RESPONSE_CALLBACK_TYPE | None] + tuple[DirectMessageParams, INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE | None] ] = [] self._data_plane_manager = DataPlaneManager(self._hierarchy, config.data_stores) @@ -444,7 +448,8 @@ def get_blocked_keys(self) -> set[str]: return self._function_keys.copy() def add_startup_messages( - self, messages: list[tuple[IntersectClientMessageParams, RESPONSE_CALLBACK_TYPE | None]] + self, + messages: list[tuple[DirectMessageParams, INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE | None]], ) -> None: """Add request messages to send out to various microservices when this service starts. @@ -455,7 +460,8 @@ def add_startup_messages( self._startup_messages.extend(messages) def add_shutdown_messages( - self, messages: list[tuple[IntersectClientMessageParams, RESPONSE_CALLBACK_TYPE | None]] + self, + messages: list[tuple[DirectMessageParams, INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE | None]], ) -> None: """Add request messages to send out to various microservices on shutdown. @@ -468,10 +474,10 @@ def add_shutdown_messages( @validate_call(config=ConfigDict(revalidate_instances='always')) def create_external_request( self, - request: IntersectClientMessageParams, - response_handler: RESPONSE_CALLBACK_TYPE | None = None, + request: DirectMessageParams, + response_handler: Union[INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE, None] = None, # noqa: UP007 (runtime checked annotation) ) -> UUID: - """Create an external request structure with a Condition we can wait on. + """Create an external request that we'll send to a different Service. Params: - request: the request we want to send out, encapsulated as an IntersectClientMessageParams object @@ -553,7 +559,11 @@ def _process_external_request(self, extreq: IntersectService._ExternalRequest) - logger.warning('Failed to send request!') # process the response - if extreq.response_fn is not None and extreq.error is None: + if ( + extreq.got_valid_response + and extreq.response_fn is not None + and extreq.error is None + ): extreq.response_fn(response) if cleanup_req: @@ -698,13 +708,14 @@ def _handle_client_message(self, message: UserspaceMessage) -> None: extreq.error = error_msg else: extreq.response = msg_payload + extreq.got_valid_response = True if extreq.waiting: extreq.cv.notify() else: error_msg = f'No external request found for message:\n{message}' logger.warning(error_msg) - def _send_client_message(self, request_id: UUID, params: IntersectClientMessageParams) -> bool: + def _send_client_message(self, request_id: UUID, params: DirectMessageParams) -> bool: """Send a userspace message.""" # "params" should already be validated at this stage. request = GENERIC_MESSAGE_SERIALIZER.dump_json(params.payload, warnings=False) diff --git a/src/intersect_sdk/service_callback_definitions.py b/src/intersect_sdk/service_callback_definitions.py new file mode 100644 index 0000000..4eb0c08 --- /dev/null +++ b/src/intersect_sdk/service_callback_definitions.py @@ -0,0 +1,16 @@ +"""Callback definitions used by Services and Capabilities. + +Please see shared_callback_definitions for definitions which are also used by Clients. +""" + +from typing import Callable + +from .client_callback_definitions import INTERSECT_JSON_VALUE + +INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE = Callable[[INTERSECT_JSON_VALUE], None] +"""Callback typing for the function which handles another Service's response. + +The function accepts one argument - the direct response from the other Service. Message metadata will not be included. + +This callback type should only be used on Capabilities - for client callback functions, use INTERSECT_CLIENT_RESPONSE_CALLBACK_TYPE . +""" diff --git a/src/intersect_sdk/shared_callback_definitions.py b/src/intersect_sdk/shared_callback_definitions.py new file mode 100644 index 0000000..170a058 --- /dev/null +++ b/src/intersect_sdk/shared_callback_definitions.py @@ -0,0 +1,67 @@ +"""Callback definitions shared between Services, Capabilities, and Clients.""" + +from typing import Any, Dict, List, Union + +from pydantic import BaseModel, ConfigDict, Field +from typing_extensions import Annotated, TypeAlias + +from .constants import SYSTEM_OF_SYSTEM_REGEX +from .core_definitions import IntersectDataHandler, IntersectMimeType + +INTERSECT_JSON_VALUE: TypeAlias = Union[ + List['INTERSECT_JSON_VALUE'], + Dict[str, 'INTERSECT_JSON_VALUE'], + str, + bool, + int, + float, + None, +] +""" +This is a simple type representation of JSON as a Python object. INTERSECT will automatically deserialize service payloads into one of these types. + +(Pydantic has a similar type, "JsonValue", which should be used if you desire functionality beyond type hinting. This is strictly a type hint.) +""" + + +class DirectMessageParams(BaseModel): + """These are the public-facing properties of a message which can be sent to another Service. + + This object can be used by Clients, and by Services if initiating a service-to-service request. + """ + + destination: Annotated[str, Field(pattern=SYSTEM_OF_SYSTEM_REGEX)] + """ + The destination string. You'll need to know the system-of-system representation of the Service. + + Note that this should match what you would see in the schema. + """ + + operation: str + """ + The name of the operation you want to call from the Service - this should be represented as it is in the Service's schema. + """ + + payload: Any + """ + The raw Python object you want to have serialized as the payload. + + If you want to just use the service's default value for a request (assuming it has a default value for a request), you may set this as None. + """ + + content_type: IntersectMimeType = IntersectMimeType.JSON + """ + The IntersectMimeType of your message. You'll want this to match with the ContentType of the function from the schema. + + default: IntersectMimeType.JSON + """ + + data_handler: IntersectDataHandler = IntersectDataHandler.MESSAGE + """ + The IntersectDataHandler you want to use (most people can just use IntersectDataHandler.MESSAGE here, unless your data is very large) + + default: IntersectDataHandler.MESSAGE + """ + + # pydantic config + model_config = ConfigDict(revalidate_instances='always') diff --git a/tests/unit/test_base_capability_implementation.py b/tests/unit/test_base_capability_implementation.py index 38147fa..1b33614 100644 --- a/tests/unit/test_base_capability_implementation.py +++ b/tests/unit/test_base_capability_implementation.py @@ -1,9 +1,12 @@ from __future__ import annotations from typing import Any +from uuid import UUID, uuid4 import pytest from intersect_sdk import ( + INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE, + DirectMessageParams, IntersectBaseCapabilityImplementation, IntersectEventDefinition, intersect_event, @@ -18,10 +21,23 @@ class MockObserver(IntersectEventObserver): def __init__(self) -> None: self.tracked_events: list[tuple[str, Any, str]] = [] + self.registered_requests: dict[ + UUID, + tuple[DirectMessageParams, INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE | None], + ] = {} def _on_observe_event(self, event_name: str, event_value: Any, operation: str) -> None: self.tracked_events.append((event_name, event_value, operation)) + def create_external_request( + self, + request: DirectMessageParams, + response_handler: INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE | None = None, + ) -> UUID: + request_id = uuid4() + self.registered_requests[request_id] = (request, response_handler) + return request_id + # TESTS #################### @@ -33,7 +49,7 @@ class BadClass1(IntersectBaseCapabilityImplementation): def _intersect_sdk_register_observer(self, observer: IntersectEventObserver) -> None: return super()._intersect_sdk_register_observer(observer) - assert 'BadClass1: Cannot override functions' in str(ex) + assert 'BadClass1: Attempted to override a reserved INTERSECT-SDK function' in str(ex) with pytest.raises(RuntimeError) as ex: @@ -41,7 +57,19 @@ class BadClass2(IntersectBaseCapabilityImplementation): def intersect_sdk_emit_event(self, event_name: str, event_value: Any) -> None: return super().intersect_sdk_emit_event(event_name, event_value) - assert 'BadClass2: Cannot override functions' in str(ex) + assert 'BadClass2: Attempted to override a reserved INTERSECT-SDK function' in str(ex) + + with pytest.raises(RuntimeError) as ex: + + class BadClass3(IntersectBaseCapabilityImplementation): + def intersect_sdk_call_service( + self, + request: DirectMessageParams, + response_handler: INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE | None = None, + ) -> None: + return super().intersect_sdk_call_service(request, response_handler) + + assert 'BadClass3: Attempted to override a reserved INTERSECT-SDK function' in str(ex) # Note that the ONLY thing the capability itself checks for are annotated functions. @@ -115,3 +143,41 @@ def _inner_function(self) -> None: assert len(observer.tracked_events) == 2 capability.outer_function() assert len(observer.tracked_events) == 3 + + +def test_functions_handle_requests(): + class Inner(IntersectBaseCapabilityImplementation): + def __init__(self) -> None: + super().__init__() + self.tracked_responses = [] + + @intersect_message() + def mock_request_flow(self, fake_request_value: str) -> UUID: + return self.intersect_sdk_call_service( + DirectMessageParams( + destination='fake.fake.fake.fake.fake', + operation='Fake.fake', + payload=fake_request_value, + ), + self._mock_other_service_callback, + )[0] + + def _mock_other_service_callback(self, param: INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE): + self.tracked_responses.append(param) + + # setup + observer = MockObserver() + capability = Inner() + capability._intersect_sdk_register_observer(observer) + + # mock making the request and setting up the response handler + body = 'ping' + reqid = capability.mock_request_flow(body) + assert len(observer.registered_requests) == 1 + req, res = observer.registered_requests[reqid] + assert req.payload == body + + # mock calling the response handler + res('pong') + assert len(capability.tracked_responses) == 1 + assert capability.tracked_responses[0] == 'pong' From 0af6ce20c2e7a33286ea1158ce18892d8a29e87b Mon Sep 17 00:00:00 2001 From: Lance Drane Date: Fri, 2 Aug 2024 17:17:26 -0400 Subject: [PATCH 05/10] rename 'DirectMessageParams' to 'IntersectDirectMessageParams' Signed-off-by: Lance Drane --- examples/1_hello_world/hello_client.py | 4 ++-- examples/1_hello_world_amqp/hello_client.py | 4 ++-- examples/1_hello_world_events/hello_client.py | 4 ++-- examples/2_counting/counting_client.py | 16 +++++++-------- src/intersect_sdk/__init__.py | 4 ++-- src/intersect_sdk/_internal/interfaces.py | 4 ++-- src/intersect_sdk/capability/base.py | 4 ++-- src/intersect_sdk/client.py | 4 ++-- .../client_callback_definitions.py | 4 ++-- src/intersect_sdk/service.py | 20 +++++++++++-------- .../shared_callback_definitions.py | 2 +- .../test_base_capability_implementation.py | 10 +++++----- 12 files changed, 42 insertions(+), 38 deletions(-) diff --git a/examples/1_hello_world/hello_client.py b/examples/1_hello_world/hello_client.py index 3a8a29c..315791a 100644 --- a/examples/1_hello_world/hello_client.py +++ b/examples/1_hello_world/hello_client.py @@ -2,10 +2,10 @@ from intersect_sdk import ( INTERSECT_JSON_VALUE, - DirectMessageParams, IntersectClient, IntersectClientCallback, IntersectClientConfig, + IntersectDirectMessageParams, default_intersect_lifecycle_loop, ) @@ -74,7 +74,7 @@ def simple_client_callback( you'll get a message back. """ initial_messages = [ - DirectMessageParams( + IntersectDirectMessageParams( destination='hello-organization.hello-facility.hello-system.hello-subsystem.hello-service', operation='HelloExample.say_hello_to_name', payload='hello_client', diff --git a/examples/1_hello_world_amqp/hello_client.py b/examples/1_hello_world_amqp/hello_client.py index a0063f4..6fb6e58 100644 --- a/examples/1_hello_world_amqp/hello_client.py +++ b/examples/1_hello_world_amqp/hello_client.py @@ -2,10 +2,10 @@ from intersect_sdk import ( INTERSECT_JSON_VALUE, - DirectMessageParams, IntersectClient, IntersectClientCallback, IntersectClientConfig, + IntersectDirectMessageParams, default_intersect_lifecycle_loop, ) @@ -76,7 +76,7 @@ def simple_client_callback( you'll get a message back. """ initial_messages = [ - DirectMessageParams( + IntersectDirectMessageParams( destination='hello-organization.hello-facility.hello-system.hello-subsystem.hello-service', operation='HelloExample.say_hello_to_name', payload='hello_client', diff --git a/examples/1_hello_world_events/hello_client.py b/examples/1_hello_world_events/hello_client.py index 15e47af..3e689a8 100644 --- a/examples/1_hello_world_events/hello_client.py +++ b/examples/1_hello_world_events/hello_client.py @@ -2,10 +2,10 @@ from intersect_sdk import ( INTERSECT_JSON_VALUE, - DirectMessageParams, IntersectClient, IntersectClientCallback, IntersectClientConfig, + IntersectDirectMessageParams, default_intersect_lifecycle_loop, ) @@ -95,7 +95,7 @@ def simple_event_callback( you'll get a message back. """ initial_messages = [ - DirectMessageParams( + IntersectDirectMessageParams( destination='hello-organization.hello-facility.hello-system.hello-subsystem.hello-service', operation='HelloExample.say_hello_to_name', payload='hello_client', diff --git a/examples/2_counting/counting_client.py b/examples/2_counting/counting_client.py index 7868fe3..4a35606 100644 --- a/examples/2_counting/counting_client.py +++ b/examples/2_counting/counting_client.py @@ -4,10 +4,10 @@ from intersect_sdk import ( INTERSECT_JSON_VALUE, - DirectMessageParams, IntersectClient, IntersectClientCallback, IntersectClientConfig, + IntersectDirectMessageParams, default_intersect_lifecycle_loop, ) @@ -38,7 +38,7 @@ def __init__(self) -> None: self.message_stack = [ # wait 5 seconds before stopping the counter. "Count" in response will be approx. 6 ( - DirectMessageParams( + IntersectDirectMessageParams( destination='counting-organization.counting-facility.counting-system.counting-subsystem.counting-service', operation='CountingExample.stop_count', payload=None, @@ -47,7 +47,7 @@ def __init__(self) -> None: ), # start the counter up again - it will not be 0 at this point! "Count" in response will be approx. 7 ( - DirectMessageParams( + IntersectDirectMessageParams( destination='counting-organization.counting-facility.counting-system.counting-subsystem.counting-service', operation='CountingExample.start_count', payload=None, @@ -56,7 +56,7 @@ def __init__(self) -> None: ), # reset the counter, but have it immediately start running again. "Count" in response will be approx. 10 ( - DirectMessageParams( + IntersectDirectMessageParams( destination='counting-organization.counting-facility.counting-system.counting-subsystem.counting-service', operation='CountingExample.reset_count', payload=True, @@ -65,7 +65,7 @@ def __init__(self) -> None: ), # reset the counter, but don't have it run again. "Count" in response will be approx. 6 ( - DirectMessageParams( + IntersectDirectMessageParams( destination='counting-organization.counting-facility.counting-system.counting-subsystem.counting-service', operation='CountingExample.reset_count', payload=False, @@ -74,7 +74,7 @@ def __init__(self) -> None: ), # start the counter back up. "Count" in response will be approx. 1 ( - DirectMessageParams( + IntersectDirectMessageParams( destination='counting-organization.counting-facility.counting-system.counting-subsystem.counting-service', operation='CountingExample.start_count', payload=None, @@ -83,7 +83,7 @@ def __init__(self) -> None: ), # finally, stop the counter one last time. "Count" in response will be approx. 4 ( - DirectMessageParams( + IntersectDirectMessageParams( destination='counting-organization.counting-facility.counting-system.counting-subsystem.counting-service', operation='CountingExample.stop_count', payload=None, @@ -158,7 +158,7 @@ def client_callback( # The counter will start after the initial message. # If the service is already active and counting, this may do nothing. initial_messages = [ - DirectMessageParams( + IntersectDirectMessageParams( destination='counting-organization.counting-facility.counting-system.counting-subsystem.counting-service', operation='CountingExample.start_count', payload=None, diff --git a/src/intersect_sdk/__init__.py b/src/intersect_sdk/__init__.py index 52d5f4a..52417b8 100644 --- a/src/intersect_sdk/__init__.py +++ b/src/intersect_sdk/__init__.py @@ -37,7 +37,7 @@ ) from .shared_callback_definitions import ( INTERSECT_JSON_VALUE, - DirectMessageParams, + IntersectDirectMessageParams, ) from .version import __version__, version_info, version_string @@ -52,7 +52,7 @@ 'IntersectService', 'IntersectClient', 'IntersectClientCallback', - 'DirectMessageParams', + 'IntersectDirectMessageParams', 'INTERSECT_CLIENT_RESPONSE_CALLBACK_TYPE', 'INTERSECT_CLIENT_EVENT_CALLBACK_TYPE', 'INTERSECT_JSON_VALUE', diff --git a/src/intersect_sdk/_internal/interfaces.py b/src/intersect_sdk/_internal/interfaces.py index 197cf3f..ee18fea 100644 --- a/src/intersect_sdk/_internal/interfaces.py +++ b/src/intersect_sdk/_internal/interfaces.py @@ -10,7 +10,7 @@ INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE, ) from ..shared_callback_definitions import ( - DirectMessageParams, + IntersectDirectMessageParams, ) @@ -34,7 +34,7 @@ def _on_observe_event(self, event_name: str, event_value: Any, operation: str) - @abstractmethod def create_external_request( self, - request: DirectMessageParams, + request: IntersectDirectMessageParams, response_handler: INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE | None = None, ) -> UUID: """Observed entity (capabilitiy) tells observer (i.e. service) to send an external request. diff --git a/src/intersect_sdk/capability/base.py b/src/intersect_sdk/capability/base.py index bee048f..a29abfe 100644 --- a/src/intersect_sdk/capability/base.py +++ b/src/intersect_sdk/capability/base.py @@ -18,7 +18,7 @@ INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE, ) from ..shared_callback_definitions import ( - DirectMessageParams, + IntersectDirectMessageParams, ) @@ -131,7 +131,7 @@ def intersect_sdk_emit_event(self, event_name: str, event_value: Any) -> None: @final def intersect_sdk_call_service( self, - request: DirectMessageParams, + request: IntersectDirectMessageParams, response_handler: INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE | None = None, ) -> list[UUID]: """Create an external request that we'll send to a different Service. diff --git a/src/intersect_sdk/client.py b/src/intersect_sdk/client.py index cc5fdb2..0219098 100644 --- a/src/intersect_sdk/client.py +++ b/src/intersect_sdk/client.py @@ -49,7 +49,7 @@ INTERSECT_CLIENT_EVENT_CALLBACK_TYPE, INTERSECT_CLIENT_RESPONSE_CALLBACK_TYPE, ) - from .shared_callback_definitions import DirectMessageParams + from .shared_callback_definitions import IntersectDirectMessageParams @final @@ -425,7 +425,7 @@ def _handle_client_callback(self, user_value: IntersectClientCallback | None) -> for message in validated_result.messages_to_send: self._send_userspace_message(message) - def _send_userspace_message(self, params: DirectMessageParams) -> None: + def _send_userspace_message(self, params: IntersectDirectMessageParams) -> None: """Send a userspace message, be it an initial message from the user or from the user's callback function.""" # ONE: SERIALIZE FUNCTION RESULTS # (function input should already be validated at this point) diff --git a/src/intersect_sdk/client_callback_definitions.py b/src/intersect_sdk/client_callback_definitions.py index 453d7d4..9bad1b9 100644 --- a/src/intersect_sdk/client_callback_definitions.py +++ b/src/intersect_sdk/client_callback_definitions.py @@ -9,7 +9,7 @@ from typing_extensions import Annotated, TypeAlias, final from .constants import SYSTEM_OF_SYSTEM_REGEX -from .shared_callback_definitions import DirectMessageParams +from .shared_callback_definitions import IntersectDirectMessageParams @final @@ -19,7 +19,7 @@ class IntersectClientCallback(BaseModel): If you do not return a value of this type (or None), this will be treated as an Exception and will break the pub-sub loop. """ - messages_to_send: List[DirectMessageParams] = [] # noqa: FA100 (runtime annotation) + messages_to_send: List[IntersectDirectMessageParams] = [] # noqa: FA100 (runtime annotation) """ Messages to send as a result of an event or a response from a Service. """ diff --git a/src/intersect_sdk/service.py b/src/intersect_sdk/service.py index d9ff452..32b768c 100644 --- a/src/intersect_sdk/service.py +++ b/src/intersect_sdk/service.py @@ -59,7 +59,7 @@ ) from .shared_callback_definitions import ( INTERSECT_JSON_VALUE, # noqa: TCH001 (runtime-checked annotation) - DirectMessageParams, # noqa: TCH001 (runtime-checked annotation) + IntersectDirectMessageParams, # noqa: TCH001 (runtime-checked annotation) ) from .version import version_string @@ -111,7 +111,7 @@ def __init__( self, req_id: UUID, req_name: str, - request: DirectMessageParams, + request: IntersectDirectMessageParams, response_handler: INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE | None = None, ) -> None: """Create an external request.""" @@ -223,13 +223,13 @@ def __init__( self._external_request_ctr = 0 self._startup_messages: list[ - tuple[DirectMessageParams, INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE | None] + tuple[IntersectDirectMessageParams, INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE | None] ] = [] self._resend_startup_messages = True self._sent_startup_messages = False self._shutdown_messages: list[ - tuple[DirectMessageParams, INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE | None] + tuple[IntersectDirectMessageParams, INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE | None] ] = [] self._data_plane_manager = DataPlaneManager(self._hierarchy, config.data_stores) @@ -449,7 +449,9 @@ def get_blocked_keys(self) -> set[str]: def add_startup_messages( self, - messages: list[tuple[DirectMessageParams, INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE | None]], + messages: list[ + tuple[IntersectDirectMessageParams, INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE | None] + ], ) -> None: """Add request messages to send out to various microservices when this service starts. @@ -461,7 +463,9 @@ def add_startup_messages( def add_shutdown_messages( self, - messages: list[tuple[DirectMessageParams, INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE | None]], + messages: list[ + tuple[IntersectDirectMessageParams, INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE | None] + ], ) -> None: """Add request messages to send out to various microservices on shutdown. @@ -474,7 +478,7 @@ def add_shutdown_messages( @validate_call(config=ConfigDict(revalidate_instances='always')) def create_external_request( self, - request: DirectMessageParams, + request: IntersectDirectMessageParams, response_handler: Union[INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE, None] = None, # noqa: UP007 (runtime checked annotation) ) -> UUID: """Create an external request that we'll send to a different Service. @@ -715,7 +719,7 @@ def _handle_client_message(self, message: UserspaceMessage) -> None: error_msg = f'No external request found for message:\n{message}' logger.warning(error_msg) - def _send_client_message(self, request_id: UUID, params: DirectMessageParams) -> bool: + def _send_client_message(self, request_id: UUID, params: IntersectDirectMessageParams) -> bool: """Send a userspace message.""" # "params" should already be validated at this stage. request = GENERIC_MESSAGE_SERIALIZER.dump_json(params.payload, warnings=False) diff --git a/src/intersect_sdk/shared_callback_definitions.py b/src/intersect_sdk/shared_callback_definitions.py index 170a058..fd1ff04 100644 --- a/src/intersect_sdk/shared_callback_definitions.py +++ b/src/intersect_sdk/shared_callback_definitions.py @@ -24,7 +24,7 @@ """ -class DirectMessageParams(BaseModel): +class IntersectDirectMessageParams(BaseModel): """These are the public-facing properties of a message which can be sent to another Service. This object can be used by Clients, and by Services if initiating a service-to-service request. diff --git a/tests/unit/test_base_capability_implementation.py b/tests/unit/test_base_capability_implementation.py index 1b33614..c7277b0 100644 --- a/tests/unit/test_base_capability_implementation.py +++ b/tests/unit/test_base_capability_implementation.py @@ -6,8 +6,8 @@ import pytest from intersect_sdk import ( INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE, - DirectMessageParams, IntersectBaseCapabilityImplementation, + IntersectDirectMessageParams, IntersectEventDefinition, intersect_event, intersect_message, @@ -23,7 +23,7 @@ def __init__(self) -> None: self.tracked_events: list[tuple[str, Any, str]] = [] self.registered_requests: dict[ UUID, - tuple[DirectMessageParams, INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE | None], + tuple[IntersectDirectMessageParams, INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE | None], ] = {} def _on_observe_event(self, event_name: str, event_value: Any, operation: str) -> None: @@ -31,7 +31,7 @@ def _on_observe_event(self, event_name: str, event_value: Any, operation: str) - def create_external_request( self, - request: DirectMessageParams, + request: IntersectDirectMessageParams, response_handler: INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE | None = None, ) -> UUID: request_id = uuid4() @@ -64,7 +64,7 @@ def intersect_sdk_emit_event(self, event_name: str, event_value: Any) -> None: class BadClass3(IntersectBaseCapabilityImplementation): def intersect_sdk_call_service( self, - request: DirectMessageParams, + request: IntersectDirectMessageParams, response_handler: INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE | None = None, ) -> None: return super().intersect_sdk_call_service(request, response_handler) @@ -154,7 +154,7 @@ def __init__(self) -> None: @intersect_message() def mock_request_flow(self, fake_request_value: str) -> UUID: return self.intersect_sdk_call_service( - DirectMessageParams( + IntersectDirectMessageParams( destination='fake.fake.fake.fake.fake', operation='Fake.fake', payload=fake_request_value, From f73b8f5343f57c703106fe8eca6bcab9c7d64d0c Mon Sep 17 00:00:00 2001 From: Lance Drane Date: Fri, 2 Aug 2024 17:36:51 -0400 Subject: [PATCH 06/10] update docs Signed-off-by: Lance Drane --- docs/api/callback_definitions.rst | 27 ++++++++++++++++++++++++ docs/api/client_callback_definitions.rst | 7 ------ docs/api/config.rst | 27 ++++++++++++++++++++++++ docs/api/config_client.rst | 7 ------ docs/api/config_service.rst | 7 ------ docs/api/config_shared.rst | 7 ------ docs/index.rst | 8 +++---- src/intersect_sdk/capability/base.py | 6 +++--- 8 files changed, 60 insertions(+), 36 deletions(-) create mode 100644 docs/api/callback_definitions.rst delete mode 100644 docs/api/client_callback_definitions.rst create mode 100644 docs/api/config.rst delete mode 100644 docs/api/config_client.rst delete mode 100644 docs/api/config_service.rst delete mode 100644 docs/api/config_shared.rst diff --git a/docs/api/callback_definitions.rst b/docs/api/callback_definitions.rst new file mode 100644 index 0000000..125459a --- /dev/null +++ b/docs/api/callback_definitions.rst @@ -0,0 +1,27 @@ +==================== +Callback Definitions +==================== + +Shared +====== + +.. automodule:: intersect_sdk.shared_callback_definitions + :members: + :undoc-members: + :exclude-members: model_computed_fields, model_config, model_fields + +Client +====== + +.. automodule:: intersect_sdk.client_callback_definitions + :members: + :undoc-members: + :exclude-members: model_computed_fields, model_config, model_fields + +Service +======= + +.. automodule:: intersect_sdk.service_callback_definitions + :members: + :undoc-members: + :exclude-members: model_computed_fields, model_config, model_fields diff --git a/docs/api/client_callback_definitions.rst b/docs/api/client_callback_definitions.rst deleted file mode 100644 index 6099d80..0000000 --- a/docs/api/client_callback_definitions.rst +++ /dev/null @@ -1,7 +0,0 @@ -Client Callback Definitions -=========================== - -.. automodule:: intersect_sdk.client_callback_definitions - :members: - :undoc-members: - :exclude-members: model_computed_fields, model_config, model_fields diff --git a/docs/api/config.rst b/docs/api/config.rst new file mode 100644 index 0000000..3db052d --- /dev/null +++ b/docs/api/config.rst @@ -0,0 +1,27 @@ +====== +Config +====== + +Config - Shared +=============== + +.. automodule:: intersect_sdk.config.shared + :members: + :undoc-members: + :exclude-members: model_computed_fields, model_config, model_fields + +Config - Service +================ + +.. automodule:: intersect_sdk.config.service + :members: + :undoc-members: + :exclude-members: model_computed_fields, model_config, model_fields + +Config - Client +=============== + +.. automodule:: intersect_sdk.config.client + :members: + :undoc-members: + :exclude-members: model_computed_fields, model_config, model_fields diff --git a/docs/api/config_client.rst b/docs/api/config_client.rst deleted file mode 100644 index 3a51e53..0000000 --- a/docs/api/config_client.rst +++ /dev/null @@ -1,7 +0,0 @@ -Config - Client -=============== - -.. automodule:: intersect_sdk.config.client - :members: - :undoc-members: - :exclude-members: model_computed_fields, model_config, model_fields diff --git a/docs/api/config_service.rst b/docs/api/config_service.rst deleted file mode 100644 index b41cc35..0000000 --- a/docs/api/config_service.rst +++ /dev/null @@ -1,7 +0,0 @@ -Config - Service -================ - -.. automodule:: intersect_sdk.config.service - :members: - :undoc-members: - :exclude-members: model_computed_fields, model_config, model_fields diff --git a/docs/api/config_shared.rst b/docs/api/config_shared.rst deleted file mode 100644 index 7315af1..0000000 --- a/docs/api/config_shared.rst +++ /dev/null @@ -1,7 +0,0 @@ -Config - Shared -=============== - -.. automodule:: intersect_sdk.config.shared - :members: - :undoc-members: - :exclude-members: model_computed_fields, model_config, model_fields diff --git a/docs/index.rst b/docs/index.rst index ed8406e..7bb6945 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -31,16 +31,14 @@ This site provides documentation for INTERSECT's Python SDK. See the getting sta :maxdepth: 2 :caption: API documentation - api/config_service - api/config_client - api/config_shared + api/config api/core_definitions api/schema api/capability - api/service_definitions api/service - api/client_callback_definitions + api/service_definitions api/client + api/callback_definitions api/app_lifecycle api/constants api/version diff --git a/src/intersect_sdk/capability/base.py b/src/intersect_sdk/capability/base.py index a29abfe..5731115 100644 --- a/src/intersect_sdk/capability/base.py +++ b/src/intersect_sdk/capability/base.py @@ -26,13 +26,13 @@ class IntersectBaseCapabilityImplementation: """Base class for all capabilities. EVERY capability implementation will need to extend this class. Additionally, if you redefine the constructor, - you MUST call super.__init__() . + you MUST call `super.__init__()` . """ def __init__(self) -> None: """This constructor just sets up observers. - NOTE: If you write your own constructor, you MUST call super.__init__() inside of it. The Service will throw an error if you don't. + NOTE: If you write your own constructor, you MUST call `super.__init__()` inside of it. The Service will throw an error if you don't. """ self._capability_name: str = 'InvalidCapability' """ @@ -49,7 +49,7 @@ def __init__(self) -> None: def __init_subclass__(cls) -> None: """This prevents users from overriding a few key functions. - General rule of thumb is that any function which starts with "intersect_sdk_" is a protected namespace for defining + General rule of thumb is that any function which starts with `intersect_sdk_` is a protected namespace for defining the INTERSECT-SDK public API between a capability and its observers. """ if ( From 798495845cd683303442d0aacf549a1d923f1852 Mon Sep 17 00:00:00 2001 From: Gregory Cage Date: Tue, 20 Aug 2024 15:25:23 -0400 Subject: [PATCH 07/10] Add initial example for service to service communication --- examples/4_service_to_service/__init__.py | 0 .../4_service_to_service/example_1_service.py | 78 +++++++++++++++++++ .../4_service_to_service/example_2_service.py | 62 +++++++++++++++ .../4_service_to_service/example_client.py | 78 +++++++++++++++++++ pyproject.toml | 1 + tests/e2e/test_examples.py | 7 ++ 6 files changed, 226 insertions(+) create mode 100644 examples/4_service_to_service/__init__.py create mode 100644 examples/4_service_to_service/example_1_service.py create mode 100644 examples/4_service_to_service/example_2_service.py create mode 100644 examples/4_service_to_service/example_client.py diff --git a/examples/4_service_to_service/__init__.py b/examples/4_service_to_service/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/examples/4_service_to_service/example_1_service.py b/examples/4_service_to_service/example_1_service.py new file mode 100644 index 0000000..801b1dd --- /dev/null +++ b/examples/4_service_to_service/example_1_service.py @@ -0,0 +1,78 @@ +"""First Service for example. Sends a message to service two and emits an event for the client.""" + +import logging + +from intersect_sdk import ( + HierarchyConfig, + IntersectBaseCapabilityImplementation, + IntersectDirectMessageParams, + IntersectEventDefinition, + IntersectService, + IntersectServiceConfig, + default_intersect_lifecycle_loop, + intersect_event, + intersect_message, + intersect_status, +) + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +class ExampleServiceOneCapabilityImplementation(IntersectBaseCapabilityImplementation): + """Service One Capability.""" + + @intersect_status() + def status(self) -> str: + """Basic status function which returns a hard-coded string.""" + return 'Up' + + @intersect_message() + def pass_text_to_service_2(self, text: str) -> None: + """Takes in a string parameter and sends it to service 2.""" + logger.info('maing it to service one') + msg_to_send = IntersectDirectMessageParams( + destination='example-organization.example-facility.example-system.example-subsystem.service-two', + operation='ServiceTwo.test_service', + payload=text, + ) + + # Send intersect message to another service + self.intersect_sdk_call_service(msg_to_send, self.service_2_handler) + + @intersect_event(events={'response_event': IntersectEventDefinition(event_type=str)}) + def service_2_handler(self, msg: str) -> None: + """Handles response from service two and emites the response as an event for the client.""" + logger.info('maing it to right before emitting event') + self.intersect_sdk_emit_event('response_event', f'Received Response from Service 2: {msg}') + + +if __name__ == '__main__': + from_config_file = { + 'brokers': [ + { + 'username': 'intersect_username', + 'password': 'intersect_password', + 'port': 1883, + 'protocol': 'mqtt3.1.1', + }, + ], + } + config = IntersectServiceConfig( + hierarchy=HierarchyConfig( + organization='example-organization', + facility='example-facility', + system='example-system', + subsystem='example-subsystem', + service='service-one', + ), + status_interval=30.0, + **from_config_file, + ) + capability = ExampleServiceOneCapabilityImplementation() + capability.capability_name = 'ServiceOne' + service = IntersectService([capability], config) + logger.info('Starting service one, use Ctrl+C to exit.') + default_intersect_lifecycle_loop( + service, + ) diff --git a/examples/4_service_to_service/example_2_service.py b/examples/4_service_to_service/example_2_service.py new file mode 100644 index 0000000..bd2b0ad --- /dev/null +++ b/examples/4_service_to_service/example_2_service.py @@ -0,0 +1,62 @@ +"""Second Service for example.""" + +import logging + +from intersect_sdk import ( + HierarchyConfig, + IntersectBaseCapabilityImplementation, + IntersectService, + IntersectServiceConfig, + default_intersect_lifecycle_loop, + intersect_message, + intersect_status, +) + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +class ExampleServiceTwoCapabilityImplementation(IntersectBaseCapabilityImplementation): + """Service Two Capability.""" + + @intersect_status() + def status(self) -> str: + """Basic status function which returns a hard-coded string.""" + return 'Up' + + @intersect_message + def test_service(self, text: str) -> str: + """Returns the text given along with acknowledgement.""" + logger.info('Making it to service 2') + return f'Acknowledging service one text -> {text}' + + +if __name__ == '__main__': + from_config_file = { + 'brokers': [ + { + 'username': 'intersect_username', + 'password': 'intersect_password', + 'port': 1883, + 'protocol': 'mqtt3.1.1', + }, + ], + } + config = IntersectServiceConfig( + hierarchy=HierarchyConfig( + organization='example-organization', + facility='example-facility', + system='example-system', + subsystem='example-subsystem', + service='service-two', + ), + status_interval=30.0, + **from_config_file, + ) + capability = ExampleServiceTwoCapabilityImplementation() + capability.capability_name = 'ServiceTwo' + service = IntersectService([capability], config) + logger.info('Starting service two, use Ctrl+C to exit.') + default_intersect_lifecycle_loop( + service, + ) diff --git a/examples/4_service_to_service/example_client.py b/examples/4_service_to_service/example_client.py new file mode 100644 index 0000000..b2fb6c0 --- /dev/null +++ b/examples/4_service_to_service/example_client.py @@ -0,0 +1,78 @@ +"""Client for service to service example. + +Kicks off exmaple by sending message to service one, and then +waits for an event from service one to confirm the messages were passed between the two services properly. + +""" + +import logging + +from intersect_sdk import ( + INTERSECT_JSON_VALUE, + IntersectClient, + IntersectClientCallback, + IntersectClientConfig, + IntersectDirectMessageParams, + default_intersect_lifecycle_loop, +) + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +class SampleOrchestrator: + """Simply contains an event callback for events from Service One.""" + + def event_callback( + self, _source: str, _operation: str, _event_name: str, payload: INTERSECT_JSON_VALUE + ) -> None: + """This simply prints the event from Service One to your console. + + Params: + source: the source of the response message. + operation: the name of the function we called in the original message. + _has_error: Boolean value which represents an error. + payload: Value of the response from the Service. + """ + logger.info('making it to event callback') + print(payload) + + +if __name__ == '__main__': + from_config_file = { + 'brokers': [ + { + 'username': 'intersect_username', + 'password': 'intersect_password', + 'port': 1883, + 'protocol': 'mqtt3.1.1', + }, + ], + } + + # The counter will start after the initial message. + # If the service is already active and counting, this may do nothing. + initial_messages = [ + IntersectDirectMessageParams( + destination='example-organization.example-facility.example-system.example-subsystem.service-one', + operation='ServiceOne.pass_text_to_service_2', + payload='Kicking off the example!', + ) + ] + config = IntersectClientConfig( + initial_message_event_config=IntersectClientCallback( + messages_to_send=initial_messages, + services_to_start_listening_for_events=[ + 'example-organization.example-facility.example-system.example-subsystem.service-one' + ], + ), + **from_config_file, + ) + orchestrator = SampleOrchestrator() + client = IntersectClient( + config=config, + event_callback=orchestrator.event_callback, + ) + default_intersect_lifecycle_loop( + client, + ) diff --git a/pyproject.toml b/pyproject.toml index 48d4431..a73451d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,6 +63,7 @@ test = [ test-all = "pytest tests/ --cov=src/intersect_sdk/ --cov-fail-under=80 --cov-report=html:reports/htmlcov/ --cov-report=xml:reports/coverage_report.xml --junitxml=reports/junit.xml" test-all-debug = "pytest tests/ --cov=src/intersect_sdk/ --cov-fail-under=80 --cov-report=html:reports/htmlcov/ --cov-report=xml:reports/coverage_report.xml --junitxml=reports/junit.xml -s" test-unit = "pytest tests/unit --cov=src/intersect_sdk/" +test-e2e = "pytest tests/e2e --cov=src/intersect_sdk/" lint = {composite = ["lint-format", "lint-ruff", "lint-mypy"]} lint-format = "ruff format" lint-ruff = "ruff check --fix" diff --git a/tests/e2e/test_examples.py b/tests/e2e/test_examples.py index c6af75f..c65071f 100644 --- a/tests/e2e/test_examples.py +++ b/tests/e2e/test_examples.py @@ -132,3 +132,10 @@ def test_example_3_ping_pong_events(): def test_example_3_ping_pong_events_amqp(): assert run_example_test('3_ping_pong_events_amqp') == 'ping\npong\nping\npong\n' + + +def test_example_4_service_to_service(): + assert ( + run_example_test('4_service_to_service') + == 'Received Response from Service 2: Acknowledging service one text -> Kicking off the example!' + ) From 8fb2b3c0916e77cc1db539fff5564ad57c983759 Mon Sep 17 00:00:00 2001 From: Gregory Cage Date: Tue, 20 Aug 2024 15:38:55 -0400 Subject: [PATCH 08/10] Add json schemas for service to service example --- .../example_1_service_schema.json | 174 ++++++++++++++++++ .../example_2_service_schema.json | 170 +++++++++++++++++ 2 files changed, 344 insertions(+) create mode 100644 examples/4_service_to_service/example_1_service_schema.json create mode 100644 examples/4_service_to_service/example_2_service_schema.json diff --git a/examples/4_service_to_service/example_1_service_schema.json b/examples/4_service_to_service/example_1_service_schema.json new file mode 100644 index 0000000..2cbb1d4 --- /dev/null +++ b/examples/4_service_to_service/example_1_service_schema.json @@ -0,0 +1,174 @@ +{ + "asyncapi": "2.6.0", + "x-intersect-version": "0.6.4", + "info": { + "title": "example-organization.example-facility.example-system.example-subsystem.service-one", + "version": "0.0.0", + "description": "Service One Capability.\n" + }, + "defaultContentType": "application/json", + "channels": { + "pass_text_to_service_2": { + "publish": { + "message": { + "schemaFormat": "application/vnd.aai.asyncapi+json;version=2.6.0", + "contentType": "application/json", + "traits": { + "$ref": "#/components/messageTraits/commonHeaders" + }, + "payload": { + "type": "null" + } + }, + "description": "Takes in a string parameter and sends it to service 2." + }, + "subscribe": { + "message": { + "schemaFormat": "application/vnd.aai.asyncapi+json;version=2.6.0", + "contentType": "application/json", + "traits": { + "$ref": "#/components/messageTraits/commonHeaders" + }, + "payload": { + "type": "string" + } + }, + "description": "Takes in a string parameter and sends it to service 2." + }, + "events": [] + } + }, + "events": { + "response_event": { + "type": "string" + } + }, + "components": { + "schemas": {}, + "messageTraits": { + "commonHeaders": { + "messageHeaders": { + "$defs": { + "IntersectDataHandler": { + "description": "What data transfer type do you want to use for handling the request/response?\n\nDefault: MESSAGE", + "enum": [ + 0, + 1 + ], + "title": "IntersectDataHandler", + "type": "integer" + } + }, + "description": "Matches the current header definition for INTERSECT messages.\n\nALL messages should contain this header.", + "properties": { + "source": { + "description": "source of the message", + "pattern": "([-a-z0-9]+\\.)*[-a-z0-9]", + "title": "Source", + "type": "string" + }, + "destination": { + "description": "destination of the message", + "pattern": "([-a-z0-9]+\\.)*[-a-z0-9]", + "title": "Destination", + "type": "string" + }, + "created_at": { + "description": "the UTC timestamp of message creation", + "format": "date-time", + "title": "Created At", + "type": "string" + }, + "sdk_version": { + "description": "SemVer string of SDK's version, used to check for compatibility", + "pattern": "^\\d+\\.\\d+\\.\\d+$", + "title": "Sdk Version", + "type": "string" + }, + "data_handler": { + "allOf": [ + { + "$ref": "#/components/messageTraits/commonHeaders/userspaceHeaders/$defs/IntersectDataHandler" + } + ], + "default": 0, + "description": "Code signifying where data is stored." + }, + "has_error": { + "default": false, + "description": "If this value is True, the payload will contain the error message (a string)", + "title": "Has Error", + "type": "boolean" + } + }, + "required": [ + "source", + "destination", + "created_at", + "sdk_version" + ], + "title": "UserspaceMessageHeader", + "type": "object" + }, + "eventHeaders": { + "$defs": { + "IntersectDataHandler": { + "description": "What data transfer type do you want to use for handling the request/response?\n\nDefault: MESSAGE", + "enum": [ + 0, + 1 + ], + "title": "IntersectDataHandler", + "type": "integer" + } + }, + "description": "Matches the current header definition for INTERSECT messages.\n\nALL messages should contain this header.", + "properties": { + "source": { + "description": "source of the message", + "pattern": "([-a-z0-9]+\\.)*[-a-z0-9]", + "title": "Source", + "type": "string" + }, + "created_at": { + "description": "the UTC timestamp of message creation", + "format": "date-time", + "title": "Created At", + "type": "string" + }, + "sdk_version": { + "description": "SemVer string of SDK's version, used to check for compatibility", + "pattern": "^\\d+\\.\\d+\\.\\d+$", + "title": "Sdk Version", + "type": "string" + }, + "data_handler": { + "allOf": [ + { + "$ref": "#/components/messageTraits/commonHeaders/eventHeaders/$defs/IntersectDataHandler" + } + ], + "default": 0, + "description": "Code signifying where data is stored." + }, + "event_name": { + "title": "Event Name", + "type": "string" + } + }, + "required": [ + "source", + "created_at", + "sdk_version", + "event_name" + ], + "title": "EventMessageHeaders", + "type": "object" + } + } + } + }, + "status": { + "type": "string" + } + } diff --git a/examples/4_service_to_service/example_2_service_schema.json b/examples/4_service_to_service/example_2_service_schema.json new file mode 100644 index 0000000..56d4741 --- /dev/null +++ b/examples/4_service_to_service/example_2_service_schema.json @@ -0,0 +1,170 @@ +{ + "asyncapi": "2.6.0", + "x-intersect-version": "0.6.4", + "info": { + "title": "example-organization.example-facility.example-system.example-subsystem.service-two", + "version": "0.0.0", + "description": "Service Two Capability.\n" + }, + "defaultContentType": "application/json", + "channels": { + "test_service": { + "publish": { + "message": { + "schemaFormat": "application/vnd.aai.asyncapi+json;version=2.6.0", + "contentType": "application/json", + "traits": { + "$ref": "#/components/messageTraits/commonHeaders" + }, + "payload": { + "type": "string" + } + }, + "description": "Returns the text given along with acknowledgement." + }, + "subscribe": { + "message": { + "schemaFormat": "application/vnd.aai.asyncapi+json;version=2.6.0", + "contentType": "application/json", + "traits": { + "$ref": "#/components/messageTraits/commonHeaders" + }, + "payload": { + "type": "string" + } + }, + "description": "Returns the text given along with acknowledgement." + }, + "events": [] + } + }, + "events": {}, + "components": { + "schemas": {}, + "messageTraits": { + "commonHeaders": { + "messageHeaders": { + "$defs": { + "IntersectDataHandler": { + "description": "What data transfer type do you want to use for handling the request/response?\n\nDefault: MESSAGE", + "enum": [ + 0, + 1 + ], + "title": "IntersectDataHandler", + "type": "integer" + } + }, + "description": "Matches the current header definition for INTERSECT messages.\n\nALL messages should contain this header.", + "properties": { + "source": { + "description": "source of the message", + "pattern": "([-a-z0-9]+\\.)*[-a-z0-9]", + "title": "Source", + "type": "string" + }, + "destination": { + "description": "destination of the message", + "pattern": "([-a-z0-9]+\\.)*[-a-z0-9]", + "title": "Destination", + "type": "string" + }, + "created_at": { + "description": "the UTC timestamp of message creation", + "format": "date-time", + "title": "Created At", + "type": "string" + }, + "sdk_version": { + "description": "SemVer string of SDK's version, used to check for compatibility", + "pattern": "^\\d+\\.\\d+\\.\\d+$", + "title": "Sdk Version", + "type": "string" + }, + "data_handler": { + "allOf": [ + { + "$ref": "#/components/messageTraits/commonHeaders/userspaceHeaders/$defs/IntersectDataHandler" + } + ], + "default": 0, + "description": "Code signifying where data is stored." + }, + "has_error": { + "default": false, + "description": "If this value is True, the payload will contain the error message (a string)", + "title": "Has Error", + "type": "boolean" + } + }, + "required": [ + "source", + "destination", + "created_at", + "sdk_version" + ], + "title": "UserspaceMessageHeader", + "type": "object" + }, + "eventHeaders": { + "$defs": { + "IntersectDataHandler": { + "description": "What data transfer type do you want to use for handling the request/response?\n\nDefault: MESSAGE", + "enum": [ + 0, + 1 + ], + "title": "IntersectDataHandler", + "type": "integer" + } + }, + "description": "Matches the current header definition for INTERSECT messages.\n\nALL messages should contain this header.", + "properties": { + "source": { + "description": "source of the message", + "pattern": "([-a-z0-9]+\\.)*[-a-z0-9]", + "title": "Source", + "type": "string" + }, + "created_at": { + "description": "the UTC timestamp of message creation", + "format": "date-time", + "title": "Created At", + "type": "string" + }, + "sdk_version": { + "description": "SemVer string of SDK's version, used to check for compatibility", + "pattern": "^\\d+\\.\\d+\\.\\d+$", + "title": "Sdk Version", + "type": "string" + }, + "data_handler": { + "allOf": [ + { + "$ref": "#/components/messageTraits/commonHeaders/eventHeaders/$defs/IntersectDataHandler" + } + ], + "default": 0, + "description": "Code signifying where data is stored." + }, + "event_name": { + "title": "Event Name", + "type": "string" + } + }, + "required": [ + "source", + "created_at", + "sdk_version", + "event_name" + ], + "title": "EventMessageHeaders", + "type": "object" + } + } + } + }, + "status": { + "type": "string" + } + } From 2ca43b7d3e55517b82eecc5181d6c2d793191c82 Mon Sep 17 00:00:00 2001 From: Gregory Cage Date: Wed, 21 Aug 2024 10:56:57 -0400 Subject: [PATCH 09/10] Remove debugging statements and fix typos --- examples/4_service_to_service/example_1_service.py | 8 +++----- examples/4_service_to_service/example_2_service.py | 5 ++--- examples/4_service_to_service/example_client.py | 5 ++--- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/examples/4_service_to_service/example_1_service.py b/examples/4_service_to_service/example_1_service.py index 801b1dd..0619e83 100644 --- a/examples/4_service_to_service/example_1_service.py +++ b/examples/4_service_to_service/example_1_service.py @@ -20,7 +20,7 @@ class ExampleServiceOneCapabilityImplementation(IntersectBaseCapabilityImplementation): - """Service One Capability.""" + """Service 1 Capability.""" @intersect_status() def status(self) -> str: @@ -30,7 +30,6 @@ def status(self) -> str: @intersect_message() def pass_text_to_service_2(self, text: str) -> None: """Takes in a string parameter and sends it to service 2.""" - logger.info('maing it to service one') msg_to_send = IntersectDirectMessageParams( destination='example-organization.example-facility.example-system.example-subsystem.service-two', operation='ServiceTwo.test_service', @@ -42,8 +41,7 @@ def pass_text_to_service_2(self, text: str) -> None: @intersect_event(events={'response_event': IntersectEventDefinition(event_type=str)}) def service_2_handler(self, msg: str) -> None: - """Handles response from service two and emites the response as an event for the client.""" - logger.info('maing it to right before emitting event') + """Handles response from service 2 and emits the response as an event for the client.""" self.intersect_sdk_emit_event('response_event', f'Received Response from Service 2: {msg}') @@ -72,7 +70,7 @@ def service_2_handler(self, msg: str) -> None: capability = ExampleServiceOneCapabilityImplementation() capability.capability_name = 'ServiceOne' service = IntersectService([capability], config) - logger.info('Starting service one, use Ctrl+C to exit.') + logger.info('Starting Service 1, use Ctrl+C to exit.') default_intersect_lifecycle_loop( service, ) diff --git a/examples/4_service_to_service/example_2_service.py b/examples/4_service_to_service/example_2_service.py index bd2b0ad..ef5670f 100644 --- a/examples/4_service_to_service/example_2_service.py +++ b/examples/4_service_to_service/example_2_service.py @@ -17,7 +17,7 @@ class ExampleServiceTwoCapabilityImplementation(IntersectBaseCapabilityImplementation): - """Service Two Capability.""" + """Service 2 Capability.""" @intersect_status() def status(self) -> str: @@ -27,7 +27,6 @@ def status(self) -> str: @intersect_message def test_service(self, text: str) -> str: """Returns the text given along with acknowledgement.""" - logger.info('Making it to service 2') return f'Acknowledging service one text -> {text}' @@ -56,7 +55,7 @@ def test_service(self, text: str) -> str: capability = ExampleServiceTwoCapabilityImplementation() capability.capability_name = 'ServiceTwo' service = IntersectService([capability], config) - logger.info('Starting service two, use Ctrl+C to exit.') + logger.info('Starting Service 2, use Ctrl+C to exit.') default_intersect_lifecycle_loop( service, ) diff --git a/examples/4_service_to_service/example_client.py b/examples/4_service_to_service/example_client.py index b2fb6c0..dd3834b 100644 --- a/examples/4_service_to_service/example_client.py +++ b/examples/4_service_to_service/example_client.py @@ -21,12 +21,12 @@ class SampleOrchestrator: - """Simply contains an event callback for events from Service One.""" + """Simply contains an event callback for events from Service 1.""" def event_callback( self, _source: str, _operation: str, _event_name: str, payload: INTERSECT_JSON_VALUE ) -> None: - """This simply prints the event from Service One to your console. + """This simply prints the event from Service 1 to your console. Params: source: the source of the response message. @@ -34,7 +34,6 @@ def event_callback( _has_error: Boolean value which represents an error. payload: Value of the response from the Service. """ - logger.info('making it to event callback') print(payload) From ccca7862f22be9eb3e18618c0d67906a31e3a65d Mon Sep 17 00:00:00 2001 From: Lance Drane Date: Wed, 21 Aug 2024 11:33:06 -0400 Subject: [PATCH 10/10] fix svc2svc request cleanup issue and E2E test Signed-off-by: Lance Drane --- .../4_service_to_service/example_client.py | 2 ++ src/intersect_sdk/service.py | 28 ++++++++++--------- tests/e2e/test_examples.py | 2 +- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/examples/4_service_to_service/example_client.py b/examples/4_service_to_service/example_client.py index dd3834b..25e2573 100644 --- a/examples/4_service_to_service/example_client.py +++ b/examples/4_service_to_service/example_client.py @@ -35,6 +35,8 @@ def event_callback( payload: Value of the response from the Service. """ print(payload) + # break out of pubsub loop + raise Exception if __name__ == '__main__': diff --git a/src/intersect_sdk/service.py b/src/intersect_sdk/service.py index 32b768c..5bdd9ae 100644 --- a/src/intersect_sdk/service.py +++ b/src/intersect_sdk/service.py @@ -125,6 +125,8 @@ def __init__( self.got_valid_response: bool = False self.response_fn = response_handler self.waiting: bool = False + self.cleanup_req = False + """When this flag is set to True, mark this request for GC deletion.""" def __init__( self, @@ -507,14 +509,6 @@ def create_external_request( self._external_requests_lock.release_lock() return request_uuid - def _delete_external_request(self, req_id: UUID) -> None: - req_id_str = str(req_id) - if req_id_str in self._external_requests: - self._external_requests_lock.acquire_lock(blocking=True) - req: IntersectService._ExternalRequest = self._external_requests.pop(req_id_str) - del req - self._external_requests_lock.release_lock() - def _get_external_request(self, req_id: UUID) -> IntersectService._ExternalRequest | None: req_id_str = str(req_id) if req_id_str in self._external_requests: @@ -524,14 +518,25 @@ def _get_external_request(self, req_id: UUID) -> IntersectService._ExternalReque def _process_external_requests(self) -> None: self._external_requests_lock.acquire_lock(blocking=True) + + # process requests for extreq in self._external_requests.values(): if not extreq.processed: self._process_external_request(extreq) + # delete requests + cleanup_list = [ + str(extreq.request_id) + for extreq in self._external_requests.values() + if extreq.cleanup_req + ] + for extreq_id in cleanup_list: + extreq = self._external_requests.pop(extreq_id) + del extreq + self._external_requests_lock.release_lock() def _process_external_request(self, extreq: IntersectService._ExternalRequest) -> None: response = None - cleanup_req = False now = datetime.now(timezone.utc) logger.debug(f'Processing external request {extreq.request_id} @ {now}') @@ -555,7 +560,7 @@ def _process_external_request(self, extreq: IntersectService._ExternalRequest) - logger.warning( f'External service request encountered an error: {error_msg}' ) - cleanup_req = True + extreq.cleanup_req = True else: logger.debug('Request wait timed-out!') extreq.waiting = False @@ -570,9 +575,6 @@ def _process_external_request(self, extreq: IntersectService._ExternalRequest) - ): extreq.response_fn(response) - if cleanup_req: - self._delete_external_request(extreq.request_id) - def _handle_service_message_raw(self, raw: bytes) -> None: """Main broker callback function. diff --git a/tests/e2e/test_examples.py b/tests/e2e/test_examples.py index c65071f..f1990df 100644 --- a/tests/e2e/test_examples.py +++ b/tests/e2e/test_examples.py @@ -137,5 +137,5 @@ def test_example_3_ping_pong_events_amqp(): def test_example_4_service_to_service(): assert ( run_example_test('4_service_to_service') - == 'Received Response from Service 2: Acknowledging service one text -> Kicking off the example!' + == 'Received Response from Service 2: Acknowledging service one text -> Kicking off the example!\n' )