From b21676acd70aa5074aabd66e04edc3f3184b034a Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 5 Mar 2024 11:50:57 +0000 Subject: [PATCH 001/169] REF: Factor out making minimal dictionary --- octue/cloud/emulators/_pub_sub.py | 9 ++------- octue/cloud/pub_sub/service.py | 14 +++----------- octue/utils/dictionaries.py | 13 +++++++++++++ pyproject.toml | 2 +- 4 files changed, 19 insertions(+), 19 deletions(-) create mode 100644 octue/utils/dictionaries.py diff --git a/octue/cloud/emulators/_pub_sub.py b/octue/cloud/emulators/_pub_sub.py index be9331164..f4ace206e 100644 --- a/octue/cloud/emulators/_pub_sub.py +++ b/octue/cloud/emulators/_pub_sub.py @@ -8,6 +8,7 @@ from octue.cloud.pub_sub.service import ANSWERS_NAMESPACE, PARENT_SENDER_TYPE, Service from octue.cloud.service_id import convert_service_id_to_pub_sub_form from octue.resources import Manifest +from octue.utils.dictionaries import make_minimal_dictionary from octue.utils.encoders import OctueJSONEncoder @@ -355,19 +356,13 @@ def ask( subscription_name = ".".join((convert_service_id_to_pub_sub_form(service_id), ANSWERS_NAMESPACE, question_uuid)) SUBSCRIPTIONS["octue.services." + subscription_name].pop(0) - question = {"kind": "question"} - - if input_values is not None: - question["input_values"] = input_values + question = make_minimal_dictionary(kind="question", input_values=input_values, children=children) # Ignore any errors from the answering service as they will be raised on the remote service in practice, not # locally as is done in this mock. if input_manifest is not None: question["input_manifest"] = input_manifest.to_primitive() - if children is not None: - question["children"] = children - try: self.children[service_id].answer( MockMessage.from_primitive( diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index 5f6c27f70..7b54c1b0d 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -29,6 +29,7 @@ ) from octue.cloud.validation import raise_if_event_is_invalid from octue.compatibility import warn_if_incompatible +from octue.utils.dictionaries import make_minimal_dictionary from octue.utils.encoders import OctueJSONEncoder from octue.utils.exceptions import convert_exception_to_primitives from octue.utils.threads import RepeatingTimer @@ -248,10 +249,7 @@ def answer(self, question, heartbeat_interval=120, timeout=30): save_diagnostics=save_diagnostics, ) - result = {"kind": "result"} - - if analysis.output_values is not None: - result["output_values"] = analysis.output_values + result = make_minimal_dictionary(kind="result", output_values=analysis.output_values) if analysis.output_manifest is not None: result["output_manifest"] = analysis.output_manifest.to_primitive() @@ -345,18 +343,12 @@ def ask( ) answer_subscription.create(allow_existing=False) - question = {"kind": "question"} - - if input_values is not None: - question["input_values"] = input_values + question = make_minimal_dictionary(kind="question", input_values=input_values, children=children) if input_manifest is not None: input_manifest.use_signed_urls_for_datasets() question["input_manifest"] = input_manifest.to_primitive() - if children is not None: - question["children"] = children - self._send_message( message=question, topic=topic, diff --git a/octue/utils/dictionaries.py b/octue/utils/dictionaries.py new file mode 100644 index 000000000..6fee8c433 --- /dev/null +++ b/octue/utils/dictionaries.py @@ -0,0 +1,13 @@ +def make_minimal_dictionary(**kwargs): + """Make a dictionary with only the keyword arguments that have a non-`None` value. + + :param kwargs: any number of key-value pairs as keyword arguments + :return dict: a dictionary containing only the key-value pairs that have a non-`None` value + """ + data = {} + + for key, value in kwargs.items(): + if value is not None: + data[key] = value + + return data diff --git a/pyproject.toml b/pyproject.toml index 0aed7fb40..b8ab6ab10 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "octue" -version = "0.52.1" +version = "0.53.0" description = "A package providing template applications for data services, and a python SDK to the Octue API." readme = "README.md" authors = ["Marcus Lugg ", "Thomas Clark "] From a5ee14a4ce94ee37d5b565d1d36872bd3c248da0 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 5 Mar 2024 12:06:48 +0000 Subject: [PATCH 002/169] DOC: Improve `create-push-subscription` CLI command help text --- octue/cli.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/octue/cli.py b/octue/cli.py index 643209551..96ee31cb0 100644 --- a/octue/cli.py +++ b/octue/cli.py @@ -366,8 +366,9 @@ def create_push_subscription( expiration_time, revision_tag, ): - """Create a push subscription on Google Pub/Sub from the Octue service to the push endpoint. If a corresponding - topic doesn't exist, it will be created. The subscription name is printed on completion. + """Create a Google Pub/Sub push subscription for an Octue service for it to receive questions from (i.e. events with + `sender_type=PARENT`). If a corresponding topic doesn't exist, it will be created. The subscription name is printed + on completion. PROJECT_NAME is the name of the Google Cloud project in which the subscription will be created From f05b10d4762b9ec7a87757f7f608d963fab015b8 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 5 Mar 2024 12:32:04 +0000 Subject: [PATCH 003/169] REF: Factor out creating push subscription skipci --- octue/cli.py | 36 +++++++++-------------------- octue/cloud/pub_sub/__init__.py | 41 +++++++++++++++++++++++++++++++++ tests/test_cli.py | 4 ++-- 3 files changed, 54 insertions(+), 27 deletions(-) diff --git a/octue/cli.py b/octue/cli.py index 96ee31cb0..02b5f8dee 100644 --- a/octue/cli.py +++ b/octue/cli.py @@ -10,10 +10,9 @@ import click from google import auth -from octue.cloud import storage -from octue.cloud.pub_sub import Subscription, Topic +from octue.cloud import pub_sub, storage from octue.cloud.pub_sub.service import PARENT_SENDER_TYPE, Service -from octue.cloud.service_id import convert_service_id_to_pub_sub_form, create_sruid, get_sruid_parts +from octue.cloud.service_id import create_sruid, get_sruid_parts from octue.cloud.storage import GoogleCloudStorageClient from octue.configuration import load_service_and_app_configuration from octue.definitions import MANIFEST_FILENAME, VALUES_FILENAME @@ -366,9 +365,8 @@ def create_push_subscription( expiration_time, revision_tag, ): - """Create a Google Pub/Sub push subscription for an Octue service for it to receive questions from (i.e. events with - `sender_type=PARENT`). If a corresponding topic doesn't exist, it will be created. The subscription name is printed - on completion. + """Create a Google Pub/Sub push subscription for an Octue service for it to receive questions from parents. If a + corresponding topic doesn't exist, it will be created. The subscription name is printed on completion. PROJECT_NAME is the name of the Google Cloud project in which the subscription will be created @@ -379,29 +377,17 @@ def create_push_subscription( PUSH_ENDPOINT is the HTTP/HTTPS endpoint of the service to push to. It should be fully formed and include the 'https://' prefix """ - service_sruid = create_sruid(namespace=service_namespace, name=service_name, revision_tag=revision_tag) - pub_sub_sruid = convert_service_id_to_pub_sub_form(service_sruid) + sruid = create_sruid(namespace=service_namespace, name=service_name, revision_tag=revision_tag) - topic = Topic(name=pub_sub_sruid, project_name=project_name) - topic.create(allow_existing=True) - - if expiration_time: - expiration_time = float(expiration_time) - else: - # Convert empty string to `None`. - expiration_time = None - - subscription = Subscription( - name=pub_sub_sruid, - topic=topic, - project_name=project_name, - filter=f'attributes.sender_type = "{PARENT_SENDER_TYPE}"', + pub_sub.create_push_subscription( + project_name, + sruid, + push_endpoint, expiration_time=expiration_time, - push_endpoint=push_endpoint, + sender_type=PARENT_SENDER_TYPE, ) - subscription.create() - click.echo(service_sruid) + click.echo(sruid) def _add_monitor_message_to_file(path, monitor_message): diff --git a/octue/cloud/pub_sub/__init__.py b/octue/cloud/pub_sub/__init__.py index bdb8ca04f..4636f0593 100644 --- a/octue/cloud/pub_sub/__init__.py +++ b/octue/cloud/pub_sub/__init__.py @@ -1,5 +1,46 @@ +from octue.cloud.service_id import convert_service_id_to_pub_sub_form + from .subscription import Subscription from .topic import Topic __all__ = ["Subscription", "Topic"] + + +PARENT_SENDER_TYPE = "PARENT" +CHILD_SENDER_TYPE = "CHILD" +VALID_SENDER_TYPES = {PARENT_SENDER_TYPE, CHILD_SENDER_TYPE} + + +def create_push_subscription(project_name, sruid, push_endpoint, sender_type, expiration_time=None): + """Create a Google Pub/Sub push subscription. If a corresponding topic doesn't exist, it will be created. + + :param str project_name: the name of the Google Cloud project in which the subscription will be created + :param str sruid: the SRUID (service revision unique identifier) + :param str push_endpoint: the HTTP/HTTPS endpoint of the service to push to. It should be fully formed and include the 'https://' prefix + :param str sender_type: the type of event to subscribe to (must be one of "PARENT" or "CHILD") + :param float|None expiration_time: the number of seconds of inactivity after which the subscription should expire. If not provided, no expiration time is applied to the subscription + """ + if sender_type not in VALID_SENDER_TYPES: + raise ValueError(f"`sender_type` must be one of {VALID_SENDER_TYPES!r}; received {sender_type!r}") + + pub_sub_sruid = convert_service_id_to_pub_sub_form(sruid) + + topic = Topic(name=pub_sub_sruid, project_name=project_name) + topic.create(allow_existing=True) + + if expiration_time: + expiration_time = float(expiration_time) + else: + expiration_time = None + + subscription = Subscription( + name=pub_sub_sruid, + topic=topic, + project_name=project_name, + filter=f'attributes.sender_type = "{sender_type}"', + expiration_time=expiration_time, + push_endpoint=push_endpoint, + ) + + subscription.create() diff --git a/tests/test_cli.py b/tests/test_cli.py index a4e553b81..aa07c71e5 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -402,8 +402,8 @@ def test_create_push_subscription(self): (["--expiration-time=100"], 100), ): with self.subTest(expiration_time_option=expiration_time_option): - with patch("octue.cli.Topic", new=MockTopic): - with patch("octue.cli.Subscription") as mock_subscription: + with patch("octue.cloud.pub_sub.Topic", new=MockTopic): + with patch("octue.cloud.pub_sub.Subscription") as mock_subscription: result = CliRunner().invoke( octue_cli, [ From 8049a3cf89a9e687395cf2b7a7b9eb0d63ef3c24 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 5 Mar 2024 15:10:18 +0000 Subject: [PATCH 004/169] FEA: Add ability to create BigQuery subscriptions --- octue/cloud/pub_sub/subscription.py | 31 ++++++++++-- octue/exceptions.py | 6 +++ tests/cloud/pub_sub/test_subscription.py | 60 +++++++++++++++++++++++- 3 files changed, 90 insertions(+), 7 deletions(-) diff --git a/octue/cloud/pub_sub/subscription.py b/octue/cloud/pub_sub/subscription.py index 560f7da42..31fc3a060 100644 --- a/octue/cloud/pub_sub/subscription.py +++ b/octue/cloud/pub_sub/subscription.py @@ -5,6 +5,7 @@ from google.protobuf.duration_pb2 import Duration # noqa from google.protobuf.field_mask_pb2 import FieldMask # noqa from google.pubsub_v1.types.pubsub import ( + BigQueryConfig, ExpirationPolicy, PushConfig, RetryPolicy, @@ -13,6 +14,7 @@ ) from octue.cloud.service_id import OCTUE_SERVICES_NAMESPACE +from octue.exceptions import ConflictingSubscriptionType logger = logging.getLogger(__name__) @@ -34,7 +36,8 @@ class Subscription: :param int|float|None expiration_time: number of seconds of inactivity after which the subscription is deleted (infinite time if `None`) :param float minimum_retry_backoff: minimum number of seconds after the acknowledgement deadline has passed to exponentially retry delivering a message to the subscription :param float maximum_retry_backoff: maximum number of seconds after the acknowledgement deadline has passed to exponentially retry delivering a message to the subscription - :param str|None push_endpoint: if this is a push subscription, this is the URL to which messages should be pushed; leave as `None` if this is a pull subscription + :param str|None push_endpoint: if this is a push subscription, this is the URL to which messages should be pushed; leave as `None` if it's not a push subscription + :param str|None bigquery_table_id: if this is a BigQuery subscription, this is the ID of the table to which messages should be written (e.g. "your-project.your-dataset.your-table"); leave as `None` if it's not a BigQuery subscription :return None: """ @@ -50,6 +53,7 @@ def __init__( minimum_retry_backoff=10, maximum_retry_backoff=600, push_endpoint=None, + bigquery_table_id=None, ): if not name.startswith(OCTUE_SERVICES_NAMESPACE): self.name = f"{OCTUE_SERVICES_NAMESPACE}.{name}" @@ -74,7 +78,14 @@ def __init__( maximum_backoff=Duration(seconds=maximum_retry_backoff), ) + if push_endpoint and bigquery_table_id: + raise ConflictingSubscriptionType( + f"A subscription can only have one of `push_endpoint` and `bigquery_table_id`; received " + f"`push_endpoint={push_endpoint!r}` and `bigquery_table_id={bigquery_table_id!r}`." + ) + self.push_endpoint = push_endpoint + self.bigquery_table_id = bigquery_table_id self._subscriber = SubscriberClient() self._created = False @@ -93,7 +104,7 @@ def is_pull_subscription(self): :return bool: """ - return self.push_endpoint is None + return (self.push_endpoint is None) and (self.bigquery_table_id is None) @property def is_push_subscription(self): @@ -103,6 +114,14 @@ def is_push_subscription(self): """ return self.push_endpoint is not None + @property + def is_bigquery_subscription(self): + """Return `True` if this is a BigQuery subscription. + + :return bool: + """ + return self.bigquery_table_id is not None + def __repr__(self): """Represent the subscription as a string. @@ -184,9 +203,11 @@ def _create_proto_message_subscription(self): :return google.pubsub_v1.types.pubsub.Subscription: """ if self.push_endpoint: - push_config = {"push_config": PushConfig(mapping=None, push_endpoint=self.push_endpoint)} # noqa + options = {"push_config": PushConfig(mapping=None, push_endpoint=self.push_endpoint)} # noqa + elif self.bigquery_table_id: + options = {"bigquery_config": BigQueryConfig(table=self.bigquery_table_id, write_metadata=True)} # noqa else: - push_config = {} + options = {} return _Subscription( mapping=None, @@ -197,7 +218,7 @@ def _create_proto_message_subscription(self): message_retention_duration=self.message_retention_duration, # noqa expiration_policy=self.expiration_policy, # noqa retry_policy=self.retry_policy, # noqa - **push_config, + **options, ) def _log_creation(self): diff --git a/octue/exceptions.py b/octue/exceptions.py index 8b31a5478..8872f8bb0 100644 --- a/octue/exceptions.py +++ b/octue/exceptions.py @@ -118,5 +118,11 @@ class PushSubscriptionCannotBePulled(OctueSDKException): """Raise if attempting to pull a push subscription.""" +class ConflictingSubscriptionType(OctueSDKException): + """Raise if attempting to instantiate a subscription that's a push subscription and BigQuery subscription at the + same time. + """ + + class ReadOnlyResource(OctueSDKException): """Raise if attempting to alter a read-only resource.""" diff --git a/tests/cloud/pub_sub/test_subscription.py b/tests/cloud/pub_sub/test_subscription.py index 433f984b7..0799714d2 100644 --- a/tests/cloud/pub_sub/test_subscription.py +++ b/tests/cloud/pub_sub/test_subscription.py @@ -6,6 +6,7 @@ from octue.cloud.emulators._pub_sub import MockSubscriber, MockSubscriptionCreationResponse from octue.cloud.pub_sub.subscription import THIRTY_ONE_DAYS, Subscription from octue.cloud.pub_sub.topic import Topic +from octue.exceptions import ConflictingSubscriptionType from tests import TEST_PROJECT_NAME from tests.base import BaseTestCase @@ -92,6 +93,22 @@ def test_create_pull_subscription(self): self.assertEqual(response._pb.retry_policy.minimum_backoff.seconds, 10) self.assertEqual(response._pb.retry_policy.maximum_backoff.seconds, 600) + def test_error_raised_if_attempting_to_create_push_subscription_at_same_time_as_bigquery_subscription(self): + """Test that an error is raised if attempting to create a subscription that's both a push subscription and a + BigQuery subscription. + """ + project_name = os.environ["TEST_PROJECT_NAME"] + topic = Topic(name="my-topic", project_name=project_name) + + with self.assertRaises(ConflictingSubscriptionType): + Subscription( + name="world", + topic=topic, + project_name=project_name, + push_endpoint="https://example.com/endpoint", + bigquery_table_id="my-project.my-dataset.my-table", + ) + def test_create_push_subscription(self): """Test that creating a push subscription works properly.""" project_name = os.environ["TEST_PROJECT_NAME"] @@ -114,13 +131,38 @@ def test_create_push_subscription(self): self.assertEqual(response._pb.retry_policy.maximum_backoff.seconds, 600) self.assertEqual(response._pb.push_config.push_endpoint, "https://example.com/endpoint") + def test_create_bigquery_subscription(self): + """Test that creating a BigQuery subscription works properly.""" + project_name = os.environ["TEST_PROJECT_NAME"] + topic = Topic(name="my-topic", project_name=project_name) + + subscription = Subscription( + name="world", + topic=topic, + project_name=project_name, + bigquery_table_id="my-project.my-dataset.my-table", + ) + + with patch("google.pubsub_v1.SubscriberClient.create_subscription", new=MockSubscriptionCreationResponse): + response = subscription.create(allow_existing=True) + + self.assertEqual(response._pb.ack_deadline_seconds, 600) + self.assertEqual(response._pb.expiration_policy.ttl.seconds, THIRTY_ONE_DAYS) + self.assertEqual(response._pb.message_retention_duration.seconds, 600) + self.assertEqual(response._pb.retry_policy.minimum_backoff.seconds, 10) + self.assertEqual(response._pb.retry_policy.maximum_backoff.seconds, 600) + self.assertEqual(response._pb.bigquery_config.table, "my-project.my-dataset.my-table") + self.assertTrue(response._pb.bigquery_config.write_metadata) + self.assertEqual(response._pb.push_config.push_endpoint, "") + def test_is_pull_subscription(self): - """Test that `is_pull_subscription` is `True` and `is_push_subscription` is `False` for a pull subscription.""" + """Test that `is_pull_subscription` is `True` for a pull subscription.""" self.assertTrue(self.subscription.is_pull_subscription) self.assertFalse(self.subscription.is_push_subscription) + self.assertFalse(self.subscription.is_bigquery_subscription) def test_is_push_subscription(self): - """Test that `is_pull_subscription` is `False` and `is_push_subscription` is `True` for a pull subscription.""" + """Test that `is_push_subscription` is `True` for a pull subscription.""" push_subscription = Subscription( name="world", topic=self.topic, @@ -130,3 +172,17 @@ def test_is_push_subscription(self): self.assertTrue(push_subscription.is_push_subscription) self.assertFalse(push_subscription.is_pull_subscription) + self.assertFalse(push_subscription.is_bigquery_subscription) + + def test_is_bigquery_subscription(self): + """Test that `is_bigquery_subscription` is `True` for a BigQuery subscription.""" + subscription = Subscription( + name="world", + topic=self.topic, + project_name=TEST_PROJECT_NAME, + bigquery_table_id="my-project.my-dataset.my-table", + ) + + self.assertTrue(subscription.is_bigquery_subscription) + self.assertFalse(subscription.is_pull_subscription) + self.assertFalse(subscription.is_push_subscription) From b45f07af904d03b28c43f7b92afece0bf46909a1 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 5 Mar 2024 15:20:56 +0000 Subject: [PATCH 005/169] ENH: Get subscription project name from topic by default --- octue/cloud/pub_sub/service.py | 4 +- octue/cloud/pub_sub/subscription.py | 6 +-- octue/cloud/pub_sub/topic.py | 3 +- tests/cloud/pub_sub/test_logging.py | 2 +- tests/cloud/pub_sub/test_message_handler.py | 9 +--- tests/cloud/pub_sub/test_service.py | 9 +--- tests/cloud/pub_sub/test_subscription.py | 56 ++++----------------- 7 files changed, 19 insertions(+), 70 deletions(-) diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index 7b54c1b0d..83e59bcf2 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -143,7 +143,6 @@ def serve(self, timeout=None, delete_topic_and_subscription_on_exit=False, allow subscription = Subscription( name=self._pub_sub_id, topic=topic, - project_name=self.backend.project_name, filter=f'attributes.sender_type = "{PARENT_SENDER_TYPE}"', expiration_time=None, ) @@ -297,7 +296,7 @@ def ask( :param bool allow_local_files: if `True`, allow the input manifest to contain references to local files - this should only be set to `True` if the child will be able to access these local files :param str save_diagnostics: must be one of {"SAVE_DIAGNOSTICS_OFF", "SAVE_DIAGNOSTICS_ON_CRASH", "SAVE_DIAGNOSTICS_ON"}; if turned on, allow the input values and manifest (and its datasets) to be saved by the child either all the time or just if it fails while processing them :param str|None question_uuid: the UUID to use for the question if a specific one is needed; a UUID is generated if not - :param str|None push_endpoint: if answers to the question should be pushed to an endpoint, provide its URL here; if they should be pulled, leave this as `None` + :param str|None push_endpoint: if answers to the question should be pushed to an endpoint, provide its URL here (the returned subscription will be a push subscription); if not, leave this as `None` :param float|None timeout: time in seconds to keep retrying sending the question :return (octue.cloud.pub_sub.subscription.Subscription, str): the answer subscription and question UUID """ @@ -337,7 +336,6 @@ def ask( answer_subscription = Subscription( name=".".join((topic.name, ANSWERS_NAMESPACE, question_uuid)), topic=topic, - project_name=self.backend.project_name, filter=f'attributes.question_uuid = "{question_uuid}" AND attributes.sender_type = "{CHILD_SENDER_TYPE}"', push_endpoint=push_endpoint, ) diff --git a/octue/cloud/pub_sub/subscription.py b/octue/cloud/pub_sub/subscription.py index 31fc3a060..021843d3f 100644 --- a/octue/cloud/pub_sub/subscription.py +++ b/octue/cloud/pub_sub/subscription.py @@ -29,7 +29,7 @@ class Subscription: :param str name: the name of the subscription excluding "projects//subscriptions/" :param octue.cloud.pub_sub.topic.Topic topic: the topic the subscription is attached to - :param str project_name: the name of the Google Cloud project that the subscription belongs to + :param str|None project_name: the name of the Google Cloud project that the subscription belongs to; if `None`, the project name of the topic is used :param str|None filter: if provided, only receive messages matching the filter (see here for filter syntax: https://cloud.google.com/pubsub/docs/subscription-message-filter#filtering_syntax) :param int ack_deadline: the time in seconds after which, if the subscriber hasn't acknowledged a message, to retry sending it to the subscription :param int message_retention_duration: unacknowledged message retention time in seconds @@ -45,7 +45,7 @@ def __init__( self, name, topic, - project_name, + project_name=None, filter=None, ack_deadline=600, message_retention_duration=600, @@ -62,7 +62,7 @@ def __init__( self.topic = topic self.filter = filter - self.path = self.generate_subscription_path(project_name, self.name) + self.path = self.generate_subscription_path(project_name or self.topic.project_name, self.name) self.ack_deadline = ack_deadline self.message_retention_duration = Duration(seconds=message_retention_duration) diff --git a/octue/cloud/pub_sub/topic.py b/octue/cloud/pub_sub/topic.py index 4b2be732b..cb0ebda83 100644 --- a/octue/cloud/pub_sub/topic.py +++ b/octue/cloud/pub_sub/topic.py @@ -28,7 +28,8 @@ def __init__(self, name, project_name): else: self.name = name - self.path = self.generate_topic_path(project_name, self.name) + self.project_name = project_name + self.path = self.generate_topic_path(self.project_name, self.name) self.messages_published = 0 self._publisher = PublisherClient() self._created = False diff --git a/tests/cloud/pub_sub/test_logging.py b/tests/cloud/pub_sub/test_logging.py index 44afbc933..4e6e9f491 100644 --- a/tests/cloud/pub_sub/test_logging.py +++ b/tests/cloud/pub_sub/test_logging.py @@ -21,7 +21,7 @@ def test_emit(self): topic.create() question_uuid = "96d69278-44ac-4631-aeea-c90fb08a1b2b" - subscription = MockSubscription(name=f"world.answers.{question_uuid}", topic=topic, project_name="blah") + subscription = MockSubscription(name=f"world.answers.{question_uuid}", topic=topic) subscription.create() log_record = makeLogRecord({"msg": "Starting analysis."}) diff --git a/tests/cloud/pub_sub/test_message_handler.py b/tests/cloud/pub_sub/test_message_handler.py index fe10a290f..8855fd467 100644 --- a/tests/cloud/pub_sub/test_message_handler.py +++ b/tests/cloud/pub_sub/test_message_handler.py @@ -28,12 +28,7 @@ def create_mock_topic_and_subscription(): """ question_uuid = str(uuid.uuid4()) topic = MockTopic(name="my-org.my-service.1-0-0", project_name=TEST_PROJECT_NAME) - - subscription = MockSubscription( - name=f"my-org.my-service.1-0-0.answers.{question_uuid}", - topic=topic, - project_name=TEST_PROJECT_NAME, - ) + subscription = MockSubscription(name=f"my-org.my-service.1-0-0.answers.{question_uuid}", topic=topic) subscription.create() return question_uuid, topic, subscription @@ -582,7 +577,6 @@ def test_pull_and_enqueue_available_messages(self): mock_subscription = MockSubscription( name=f"my-org.my-service.1-0-0.answers.{question_uuid}", topic=mock_topic, - project_name=TEST_PROJECT_NAME, ) message_handler = OrderedMessageHandler( @@ -622,7 +616,6 @@ def test_timeout_error_raised_if_result_message_not_received_in_time(self): mock_subscription = MockSubscription( name=f"my-org.my-service.1-0-0.answers.{question_uuid}", topic=mock_topic, - project_name=TEST_PROJECT_NAME, ) message_handler = OrderedMessageHandler( diff --git a/tests/cloud/pub_sub/test_service.py b/tests/cloud/pub_sub/test_service.py index 9ae63cbfb..799b6eff2 100644 --- a/tests/cloud/pub_sub/test_service.py +++ b/tests/cloud/pub_sub/test_service.py @@ -170,13 +170,7 @@ def test_ask_service_with_no_revision_tag_when_service_registries_not_specified_ def test_timeout_error_raised_if_no_messages_received_when_waiting(self): """Test that a TimeoutError is raised if no messages are received while waiting.""" mock_topic = MockTopic(name="amazing.service.9-9-9", project_name=TEST_PROJECT_NAME) - - mock_subscription = MockSubscription( - name="amazing.service.9-9-9", - topic=mock_topic, - project_name=TEST_PROJECT_NAME, - ) - + mock_subscription = MockSubscription(name="amazing.service.9-9-9", topic=mock_topic) service = Service(backend=BACKEND) with patch("octue.cloud.pub_sub.service.pubsub_v1.SubscriberClient.pull", return_value=MockPullResponse()): @@ -188,7 +182,6 @@ def test_error_raised_if_attempting_to_wait_for_answer_from_push_subscription(se mock_subscription = MockSubscription( name="world", topic=MockTopic(name="world", project_name=TEST_PROJECT_NAME), - project_name=TEST_PROJECT_NAME, push_endpoint="https://example.com/endpoint", ) diff --git a/tests/cloud/pub_sub/test_subscription.py b/tests/cloud/pub_sub/test_subscription.py index 0799714d2..4303b8970 100644 --- a/tests/cloud/pub_sub/test_subscription.py +++ b/tests/cloud/pub_sub/test_subscription.py @@ -12,8 +12,8 @@ class TestSubscription(BaseTestCase): - topic = Topic(name="world", project_name="my-project") - subscription = Subscription(name="world", topic=topic, project_name=TEST_PROJECT_NAME) + topic = Topic(name="world", project_name=TEST_PROJECT_NAME) + subscription = Subscription(name="world", topic=topic) def test_repr(self): """Test that subscriptions are represented correctly.""" @@ -22,13 +22,7 @@ def test_repr(self): def test_namespace_only_in_name_once(self): """Test that the subscription's namespace only appears in its name once, even if it is repeated.""" self.assertEqual(self.subscription.name, "octue.services.world") - - subscription_with_repeated_namespace = Subscription( - name="octue.services.world", - topic=self.topic, - project_name=TEST_PROJECT_NAME, - ) - + subscription_with_repeated_namespace = Subscription(name="octue.services.world", topic=self.topic) self.assertEqual(subscription_with_repeated_namespace.name, "octue.services.world") def test_create_without_allow_existing_when_subscription_already_exists(self): @@ -36,7 +30,7 @@ def test_create_without_allow_existing_when_subscription_already_exists(self): `False`. """ with patch("octue.cloud.pub_sub.subscription.SubscriberClient", MockSubscriber): - subscription = Subscription(name="world", topic=self.topic, project_name=TEST_PROJECT_NAME) + subscription = Subscription(name="world", topic=self.topic) with patch( "octue.cloud.emulators._pub_sub.MockSubscriber.create_subscription", @@ -53,7 +47,7 @@ def test_create_with_allow_existing_when_already_exists(self): error. """ with patch("octue.cloud.pub_sub.subscription.SubscriberClient", MockSubscriber): - subscription = Subscription(name="world", topic=self.topic, project_name=TEST_PROJECT_NAME) + subscription = Subscription(name="world", topic=self.topic) with patch( "octue.cloud.emulators._pub_sub.MockSubscriber.create_subscription", @@ -70,13 +64,7 @@ def test_create_pull_subscription(self): """ project_name = os.environ["TEST_PROJECT_NAME"] topic = Topic(name="my-topic", project_name=project_name) - - subscription = Subscription( - name="world", - topic=topic, - project_name=project_name, - filter='attributes.question_uuid = "abc"', - ) + subscription = Subscription(name="world", topic=topic, filter='attributes.question_uuid = "abc"') for allow_existing in (True, False): with self.subTest(allow_existing=allow_existing): @@ -104,7 +92,6 @@ def test_error_raised_if_attempting_to_create_push_subscription_at_same_time_as_ Subscription( name="world", topic=topic, - project_name=project_name, push_endpoint="https://example.com/endpoint", bigquery_table_id="my-project.my-dataset.my-table", ) @@ -113,13 +100,7 @@ def test_create_push_subscription(self): """Test that creating a push subscription works properly.""" project_name = os.environ["TEST_PROJECT_NAME"] topic = Topic(name="my-topic", project_name=project_name) - - subscription = Subscription( - name="world", - topic=topic, - project_name=project_name, - push_endpoint="https://example.com/endpoint", - ) + subscription = Subscription(name="world", topic=topic, push_endpoint="https://example.com/endpoint") with patch("google.pubsub_v1.SubscriberClient.create_subscription", new=MockSubscriptionCreationResponse): response = subscription.create(allow_existing=True) @@ -135,13 +116,7 @@ def test_create_bigquery_subscription(self): """Test that creating a BigQuery subscription works properly.""" project_name = os.environ["TEST_PROJECT_NAME"] topic = Topic(name="my-topic", project_name=project_name) - - subscription = Subscription( - name="world", - topic=topic, - project_name=project_name, - bigquery_table_id="my-project.my-dataset.my-table", - ) + subscription = Subscription(name="world", topic=topic, bigquery_table_id="my-project.my-dataset.my-table") with patch("google.pubsub_v1.SubscriberClient.create_subscription", new=MockSubscriptionCreationResponse): response = subscription.create(allow_existing=True) @@ -163,25 +138,14 @@ def test_is_pull_subscription(self): def test_is_push_subscription(self): """Test that `is_push_subscription` is `True` for a pull subscription.""" - push_subscription = Subscription( - name="world", - topic=self.topic, - project_name=TEST_PROJECT_NAME, - push_endpoint="https://example.com/endpoint", - ) - + push_subscription = Subscription(name="world", topic=self.topic, push_endpoint="https://example.com/endpoint") self.assertTrue(push_subscription.is_push_subscription) self.assertFalse(push_subscription.is_pull_subscription) self.assertFalse(push_subscription.is_bigquery_subscription) def test_is_bigquery_subscription(self): """Test that `is_bigquery_subscription` is `True` for a BigQuery subscription.""" - subscription = Subscription( - name="world", - topic=self.topic, - project_name=TEST_PROJECT_NAME, - bigquery_table_id="my-project.my-dataset.my-table", - ) + subscription = Subscription(name="world", topic=self.topic, bigquery_table_id="my-project.my-dataset.my-table") self.assertTrue(subscription.is_bigquery_subscription) self.assertFalse(subscription.is_pull_subscription) From 9c785a5b07bdb816f8c0af2a0cb75fa5b3e7517b Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 5 Mar 2024 15:22:59 +0000 Subject: [PATCH 006/169] ENH: Raise error if attempting to pull a BigQuery subscription skipci --- octue/cloud/pub_sub/service.py | 8 +++++++- octue/exceptions.py | 4 ++-- tests/cloud/pub_sub/test_service.py | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index 83e59bcf2..8af1c71db 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -382,11 +382,17 @@ def wait_for_answer( :return dict: dictionary containing the keys "output_values" and "output_manifest" """ if subscription.is_push_subscription: - raise octue.exceptions.PushSubscriptionCannotBePulled( + raise octue.exceptions.NotAPullSubscription( f"{subscription.path!r} is a push subscription so it cannot be waited on for an answer. Please check " f"its push endpoint at {subscription.push_endpoint!r}." ) + if subscription.is_bigquery_subscription: + raise octue.exceptions.NotAPullSubscription( + f"{subscription.path!r} is a BigQuery subscription so it cannot be waited on for an answer. Please " + f"check its BigQuery table {subscription.bigquery_table_id!r}." + ) + service_name = get_sruid_from_pub_sub_resource_name(subscription.name) self._message_handler = OrderedMessageHandler( diff --git a/octue/exceptions.py b/octue/exceptions.py index 8872f8bb0..1e93ea037 100644 --- a/octue/exceptions.py +++ b/octue/exceptions.py @@ -114,8 +114,8 @@ class CloudStorageBucketNotFound(OctueSDKException): """Raise if attempting to access a cloud storage bucket that cannot be found.""" -class PushSubscriptionCannotBePulled(OctueSDKException): - """Raise if attempting to pull a push subscription.""" +class NotAPullSubscription(OctueSDKException): + """Raise if attempting to pull a subscription that's not a pull subscription.""" class ConflictingSubscriptionType(OctueSDKException): diff --git a/tests/cloud/pub_sub/test_service.py b/tests/cloud/pub_sub/test_service.py index 799b6eff2..ddf3d1e9e 100644 --- a/tests/cloud/pub_sub/test_service.py +++ b/tests/cloud/pub_sub/test_service.py @@ -187,7 +187,7 @@ def test_error_raised_if_attempting_to_wait_for_answer_from_push_subscription(se service = Service(backend=BACKEND) - with self.assertRaises(exceptions.PushSubscriptionCannotBePulled): + with self.assertRaises(exceptions.NotAPullSubscription): service.wait_for_answer(subscription=mock_subscription) def test_exceptions_in_responder_are_handled_and_sent_to_asker(self): From 3e9919a0981f6b1b63d3be9c929dc91fe7bd0822 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 5 Mar 2024 16:07:13 +0000 Subject: [PATCH 007/169] ENH: Add ability to ask questions with BigQuery subscription --- octue/cloud/pub_sub/service.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index 8af1c71db..84add123a 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -282,6 +282,7 @@ def ask( save_diagnostics="SAVE_DIAGNOSTICS_ON_CRASH", # This is repeated as a string here to avoid a circular import. question_uuid=None, push_endpoint=None, + bigquery_table_id=None, timeout=86400, ): """Ask a child a question (i.e. send it input values for it to analyse and produce output values for) and return @@ -297,6 +298,7 @@ def ask( :param str save_diagnostics: must be one of {"SAVE_DIAGNOSTICS_OFF", "SAVE_DIAGNOSTICS_ON_CRASH", "SAVE_DIAGNOSTICS_ON"}; if turned on, allow the input values and manifest (and its datasets) to be saved by the child either all the time or just if it fails while processing them :param str|None question_uuid: the UUID to use for the question if a specific one is needed; a UUID is generated if not :param str|None push_endpoint: if answers to the question should be pushed to an endpoint, provide its URL here (the returned subscription will be a push subscription); if not, leave this as `None` + :param str|None bigquery_table_id: if answers to the questions should be written to BigQuery, provide the ID of the table here (e.g. "your-project.your-dataset.your-table") (the returned subscription will be a BigQuery subscription); if not, leave this as `None` :param float|None timeout: time in seconds to keep retrying sending the question :return (octue.cloud.pub_sub.subscription.Subscription, str): the answer subscription and question UUID """ @@ -338,6 +340,7 @@ def ask( topic=topic, filter=f'attributes.question_uuid = "{question_uuid}" AND attributes.sender_type = "{CHILD_SENDER_TYPE}"', push_endpoint=push_endpoint, + bigquery_table_id=bigquery_table_id, ) answer_subscription.create(allow_existing=False) From ea4cda3f2767a9769a9cd715b625ee53230a5142 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 5 Mar 2024 16:20:02 +0000 Subject: [PATCH 008/169] REF: Factor out sending question message in `Service.ask` --- octue/cloud/pub_sub/service.py | 68 ++++++++++++++++++++++++++-------- 1 file changed, 53 insertions(+), 15 deletions(-) diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index 84add123a..163a15fba 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -344,24 +344,17 @@ def ask( ) answer_subscription.create(allow_existing=False) - question = make_minimal_dictionary(kind="question", input_values=input_values, children=children) - - if input_manifest is not None: - input_manifest.use_signed_urls_for_datasets() - question["input_manifest"] = input_manifest.to_primitive() - - self._send_message( - message=question, + self._send_question( + input_values=input_values, + input_manifest=input_manifest, + children=children, + service_id=service_id, + forward_logs=subscribe_to_logs, + save_diagnostics=save_diagnostics, topic=topic, - attributes={ - "question_uuid": question_uuid, - "sender_type": PARENT_SENDER_TYPE, - "forward_logs": subscribe_to_logs, - "save_diagnostics": save_diagnostics, - }, + question_uuid=question_uuid, ) - logger.info("%r asked a question %r to service %r.", self, question_uuid, service_id) return answer_subscription, question_uuid def wait_for_answer( @@ -472,6 +465,51 @@ def _send_message(self, message, topic, attributes=None, timeout=30): topic.messages_published += 1 + def _send_question( + self, + input_values, + input_manifest, + children, + service_id, + forward_logs, + save_diagnostics, + topic, + question_uuid, + timeout=30, + ): + """Send a question to a child service. + + :param any|None input_values: any input values for the question + :param octue.resources.manifest.Manifest|None input_manifest: an input manifest of any datasets needed for the question + :param list(dict)|None children: a list of children for the child to use instead of its default children (if it uses children). These should be in the same format as in an app's app configuration file and have the same keys. + :param str service_id: the ID of the child to send the question to + :param bool forward_logs: whether to request the child to forward its logs + :param str save_diagnostics: must be one of {"SAVE_DIAGNOSTICS_OFF", "SAVE_DIAGNOSTICS_ON_CRASH", "SAVE_DIAGNOSTICS_ON"}; if turned on, allow the input values and manifest (and its datasets) to be saved by the child either all the time or just if it fails while processing them + :param octue.cloud.pub_sub.topic.Topic topic: topic to send the acknowledgement to + :param str question_uuid: + :param float timeout: time in seconds after which to give up sending + :return None: + """ + question = make_minimal_dictionary(kind="question", input_values=input_values, children=children) + + if input_manifest is not None: + input_manifest.use_signed_urls_for_datasets() + question["input_manifest"] = input_manifest.to_primitive() + + self._send_message( + message=question, + topic=topic, + timeout=timeout, + attributes={ + "question_uuid": question_uuid, + "sender_type": PARENT_SENDER_TYPE, + "forward_logs": forward_logs, + "save_diagnostics": save_diagnostics, + }, + ) + + logger.info("%r asked a question %r to service %r.", self, question_uuid, service_id) + def _send_delivery_acknowledgment(self, topic, question_uuid, timeout=30): """Send an acknowledgement of question receipt to the parent. From 414699ed37b3e0f75a55eef0158b1e5b69b99c6b Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 5 Mar 2024 17:19:54 +0000 Subject: [PATCH 009/169] TST: Test error raised if trying to pull a BigQuery subscription --- tests/cloud/pub_sub/test_service.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/tests/cloud/pub_sub/test_service.py b/tests/cloud/pub_sub/test_service.py index ddf3d1e9e..e72f51419 100644 --- a/tests/cloud/pub_sub/test_service.py +++ b/tests/cloud/pub_sub/test_service.py @@ -177,18 +177,25 @@ def test_timeout_error_raised_if_no_messages_received_when_waiting(self): with self.assertRaises(TimeoutError): service.wait_for_answer(subscription=mock_subscription, timeout=0.01) - def test_error_raised_if_attempting_to_wait_for_answer_from_push_subscription(self): + def test_error_raised_if_attempting_to_wait_for_answer_from_non_pull_subscription(self): """Test that an error is raised if attempting to wait for an answer from a push subscription.""" - mock_subscription = MockSubscription( - name="world", - topic=MockTopic(name="world", project_name=TEST_PROJECT_NAME), - push_endpoint="https://example.com/endpoint", - ) - service = Service(backend=BACKEND) - with self.assertRaises(exceptions.NotAPullSubscription): - service.wait_for_answer(subscription=mock_subscription) + for subscription in [ + MockSubscription( + name="world", + topic=MockTopic(name="world", project_name=TEST_PROJECT_NAME), + push_endpoint="https://example.com/endpoint", + ), + MockSubscription( + name="world", + topic=MockTopic(name="world", project_name=TEST_PROJECT_NAME), + bigquery_table_id="some-table", + ), + ]: + with self.subTest(subscription=subscription): + with self.assertRaises(exceptions.NotAPullSubscription): + service.wait_for_answer(subscription=subscription) def test_exceptions_in_responder_are_handled_and_sent_to_asker(self): """Test that exceptions raised in the child service are handled and sent back to the asker.""" From 2cc8bbfcbd3b648163320c2e098f62362cd38f03 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 5 Mar 2024 17:42:11 +0000 Subject: [PATCH 010/169] OPS: Add test BigQuery dataset to terraform config --- terraform/bigquery.tf | 51 +++++++++++++++++++++++++++++++++++++++++++ terraform/iam.tf | 9 ++++++++ 2 files changed, 60 insertions(+) create mode 100644 terraform/bigquery.tf diff --git a/terraform/bigquery.tf b/terraform/bigquery.tf new file mode 100644 index 000000000..cd76b2cac --- /dev/null +++ b/terraform/bigquery.tf @@ -0,0 +1,51 @@ +resource "google_bigquery_dataset" "test_dataset" { + dataset_id = "octue_sdk_python_test_dataset" + description = "A dataset for testing BigQuery subscriptions for the Octue SDK." + location = "EU" + default_table_expiration_ms = 3600000 + + labels = { + env = "default" + } +} + +resource "google_bigquery_table" "test_table" { + dataset_id = google_bigquery_dataset.test_dataset.dataset_id + table_id = "question-events" + + labels = { + env = "default" + } + + schema = < Date: Tue, 5 Mar 2024 18:57:22 +0000 Subject: [PATCH 011/169] ENH: Allow asking of asynchronous questions via `Child.ask` --- octue/resources/child.py | 25 +++++++++++++------ .../cloud_run/test_cloud_run_deployment.py | 9 +++++++ 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/octue/resources/child.py b/octue/resources/child.py index 7ac6ec839..78a32b0b0 100644 --- a/octue/resources/child.py +++ b/octue/resources/child.py @@ -63,15 +63,19 @@ def ask( record_messages=True, save_diagnostics="SAVE_DIAGNOSTICS_ON_CRASH", # This is repeated as a string here to avoid a circular import. question_uuid=None, + push_endpoint=None, + bigquery_table_id=None, timeout=86400, maximum_heartbeat_interval=300, ): - """Ask the child a question and wait for its answer - i.e. send it input values and/or an input manifest and - wait for it to analyse them and return output values and/or an output manifest. The input values and manifest - must conform to the schema in the child's twine. - - :param any|None input_values: any input values for the question - :param octue.resources.manifest.Manifest|None input_manifest: an input manifest of any datasets needed for the question + """Ask the child either: + - A synchronous (ask-and-wait) question and wait for it to return an output. Questions are synchronous if + neither the `push_endpoint` or the `bigquery_table_id` argument is provided. + - An asynchronous (fire-and-forget) question and return immediately. To make a question asynchronous, provide + either the `push_endpoint` or `bigquery_table_id` argument. + + :param any|None input_values: any input values for the question, conforming with the schema in the child's twine + :param octue.resources.manifest.Manifest|None input_manifest: an input manifest of any datasets needed for the question, conforming with the schema in the child's twine :param list(dict)|None children: a list of children for the child to use instead of its default children (if it uses children). These should be in the same format as in an app's app configuration file and have the same keys. :param bool subscribe_to_logs: if `True`, subscribe to logs from the child and handle them with the local log handlers :param bool allow_local_files: if `True`, allow the input manifest to contain references to local files - this should only be set to `True` if the child will have access to these local files @@ -79,10 +83,12 @@ def ask( :param bool record_messages: if `True`, record messages received from the child in the `received_messages` property :param str save_diagnostics: must be one of {"SAVE_DIAGNOSTICS_OFF", "SAVE_DIAGNOSTICS_ON_CRASH", "SAVE_DIAGNOSTICS_ON"}; if turned on, allow the input values and manifest (and its datasets) to be saved by the child either all the time or just if it fails while processing them :param str|None question_uuid: the UUID to use for the question if a specific one is needed; a UUID is generated if not + :param str|None push_endpoint: if answers to the question should be pushed to an endpoint, provide its URL here (the returned subscription will be a push subscription); if not, leave this as `None` + :param str|None bigquery_table_id: if answers to the questions should be written to BigQuery, provide the ID of the table here (e.g. "your-project.your-dataset.your-table") (the returned subscription will be a BigQuery subscription); if not, leave this as `None` :param float timeout: time in seconds to wait for an answer before raising a timeout error :param float|int maximum_heartbeat_interval: the maximum amount of time (in seconds) allowed between child heartbeats before an error is raised :raise TimeoutError: if the timeout is exceeded while waiting for an answer - :return dict: a dictionary containing the keys "output_values" and "output_manifest" + :return dict|None: for a synchronous question, a dictionary containing the keys "output_values" and "output_manifest"; for an asynchronous question, `None` """ subscription, _ = self._service.ask( service_id=self.id, @@ -93,9 +99,14 @@ def ask( allow_local_files=allow_local_files, save_diagnostics=save_diagnostics, question_uuid=question_uuid, + push_endpoint=push_endpoint, + bigquery_table_id=bigquery_table_id, timeout=timeout, ) + if push_endpoint or bigquery_table_id: + return None + return self._service.wait_for_answer( subscription=subscription, handle_monitor_message=handle_monitor_message, diff --git a/tests/cloud/deployment/google/cloud_run/test_cloud_run_deployment.py b/tests/cloud/deployment/google/cloud_run/test_cloud_run_deployment.py index 5ede89e54..23a91297e 100644 --- a/tests/cloud/deployment/google/cloud_run/test_cloud_run_deployment.py +++ b/tests/cloud/deployment/google/cloud_run/test_cloud_run_deployment.py @@ -34,3 +34,12 @@ def test_cloud_run_deployment(self): # Check that the output dataset and its files can be accessed. with answer["output_manifest"].datasets["example_dataset"].files.one() as (datafile, f): self.assertEqual(f.read(), "This is some example service output.") + + def test_cloud_run_deployment_asynchronously(self): + """Test asking an asynchronous (BigQuery) question.""" + answer = self.child.ask( + input_values={"n_iterations": 3}, + bigquery_table_id="octue-sdk-python.octue_sdk_python_test_dataset.question-events", + ) + + self.assertIsNone(answer) From 3c3f929dba8bb7d8022d18da9589c6e4eb78f794 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 6 Mar 2024 10:58:31 +0000 Subject: [PATCH 012/169] FIX: Await successful publishing of question messages --- octue/cloud/pub_sub/service.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index 163a15fba..6eba37151 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -439,7 +439,7 @@ def _send_message(self, message, topic, attributes=None, timeout=30): :param octue.cloud.pub_sub.topic.Topic topic: the Pub/Sub topic to send the message to :param dict|None attributes: key-value pairs to attach to the message - the values must be strings or bytes :param int|float timeout: the timeout for sending the message in seconds - :return None: + :return google.cloud.pubsub_v1.publisher.futures.Future: """ attributes = attributes or {} @@ -456,7 +456,7 @@ def _send_message(self, message, topic, attributes=None, timeout=30): converted_attributes[key] = value - self.publisher.publish( + future = self.publisher.publish( topic=topic.path, data=json.dumps(message, cls=OctueJSONEncoder).encode(), retry=retry.Retry(deadline=timeout), @@ -465,6 +465,8 @@ def _send_message(self, message, topic, attributes=None, timeout=30): topic.messages_published += 1 + return future + def _send_question( self, input_values, @@ -496,7 +498,7 @@ def _send_question( input_manifest.use_signed_urls_for_datasets() question["input_manifest"] = input_manifest.to_primitive() - self._send_message( + future = self._send_message( message=question, topic=topic, timeout=timeout, @@ -508,6 +510,8 @@ def _send_question( }, ) + # Await successful publishing of the question. + future.result() logger.info("%r asked a question %r to service %r.", self, question_uuid, service_id) def _send_delivery_acknowledgment(self, topic, question_uuid, timeout=30): From 1def7eff7266afdc1dea746f082dea2bdacd0c7c Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 6 Mar 2024 11:04:39 +0000 Subject: [PATCH 013/169] DEP: Add `google-cloud-bigquery` --- poetry.lock | 32 +++++++++++++++++++++++++++++++- pyproject.toml | 1 + 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 3185461e7..8be51102c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -713,6 +713,36 @@ pyopenssl = ["cryptography (>=38.0.3)", "pyopenssl (>=20.0.0)"] reauth = ["pyu2f (>=0.1.5)"] requests = ["requests (>=2.20.0,<3.0.0.dev0)"] +[[package]] +name = "google-cloud-bigquery" +version = "3.18.0" +description = "Google BigQuery API client library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-cloud-bigquery-3.18.0.tar.gz", hash = "sha256:74f0fc6f0ba9477f808d25924dc8a052c55f7ca91064e83e16d3ee5fb7ca77ab"}, + {file = "google_cloud_bigquery-3.18.0-py2.py3-none-any.whl", hash = "sha256:3520552075502c69710d37b1e9600c84e6974ad271914677d16bfafa502857fb"}, +] + +[package.dependencies] +google-api-core = ">=1.31.5,<2.0.dev0 || >2.3.0,<3.0.0dev" +google-cloud-core = ">=1.6.0,<3.0.0dev" +google-resumable-media = ">=0.6.0,<3.0dev" +packaging = ">=20.0.0" +python-dateutil = ">=2.7.2,<3.0dev" +requests = ">=2.21.0,<3.0.0dev" + +[package.extras] +all = ["Shapely (>=1.8.4,<3.0.0dev)", "db-dtypes (>=0.3.0,<2.0.0dev)", "geopandas (>=0.9.0,<1.0dev)", "google-cloud-bigquery-storage (>=2.6.0,<3.0.0dev)", "grpcio (>=1.47.0,<2.0dev)", "grpcio (>=1.49.1,<2.0dev)", "importlib-metadata (>=1.0.0)", "ipykernel (>=6.0.0)", "ipython (>=7.23.1,!=8.1.0)", "ipywidgets (>=7.7.0)", "opentelemetry-api (>=1.1.0)", "opentelemetry-instrumentation (>=0.20b0)", "opentelemetry-sdk (>=1.1.0)", "pandas (>=1.1.0)", "proto-plus (>=1.15.0,<2.0.0dev)", "protobuf (>=3.19.5,!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev)", "pyarrow (>=3.0.0)", "tqdm (>=4.7.4,<5.0.0dev)"] +bigquery-v2 = ["proto-plus (>=1.15.0,<2.0.0dev)", "protobuf (>=3.19.5,!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev)"] +bqstorage = ["google-cloud-bigquery-storage (>=2.6.0,<3.0.0dev)", "grpcio (>=1.47.0,<2.0dev)", "grpcio (>=1.49.1,<2.0dev)", "pyarrow (>=3.0.0)"] +geopandas = ["Shapely (>=1.8.4,<3.0.0dev)", "geopandas (>=0.9.0,<1.0dev)"] +ipython = ["ipykernel (>=6.0.0)", "ipython (>=7.23.1,!=8.1.0)"] +ipywidgets = ["ipykernel (>=6.0.0)", "ipywidgets (>=7.7.0)"] +opentelemetry = ["opentelemetry-api (>=1.1.0)", "opentelemetry-instrumentation (>=0.20b0)", "opentelemetry-sdk (>=1.1.0)"] +pandas = ["db-dtypes (>=0.3.0,<2.0.0dev)", "importlib-metadata (>=1.0.0)", "pandas (>=1.1.0)", "pyarrow (>=3.0.0)"] +tqdm = ["tqdm (>=4.7.4,<5.0.0dev)"] + [[package]] name = "google-cloud-core" version = "2.4.1" @@ -2619,4 +2649,4 @@ hdf5 = ["h5py"] [metadata] lock-version = "2.0" python-versions = "^3.7.1" -content-hash = "7f257d18b92faa0a8a5acc8bb04c3b80583a03265115c049c651b3f859da431a" +content-hash = "ccaa2540a6d43f5c61d5fff4e770a526673a8e64892c8668e3a0b6339128e3ed" diff --git a/pyproject.toml b/pyproject.toml index b8ab6ab10..aaeddbfc8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ pyyaml = "^6" h5py = { version = "^3.6", optional = true } twined = "^0.5.1" packaging = ">=20.4" +google-cloud-bigquery = "^3.18.0" [tool.poetry.extras] hdf5 = ["h5py"] From cdac7bf589d0faadf02071b13626de181f24c809 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 6 Mar 2024 11:16:03 +0000 Subject: [PATCH 014/169] DEP: Upgrade `coolname` --- poetry.lock | 196 +++++++++++++++++++++++++------------------------ pyproject.toml | 2 +- 2 files changed, 100 insertions(+), 98 deletions(-) diff --git a/poetry.lock b/poetry.lock index 8be51102c..fe52a6400 100644 --- a/poetry.lock +++ b/poetry.lock @@ -230,13 +230,13 @@ redis = ["redis (>=2.10.5)"] [[package]] name = "cachetools" -version = "5.3.2" +version = "5.3.3" description = "Extensible memoizing collections and decorators" optional = false python-versions = ">=3.7" files = [ - {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, - {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, + {file = "cachetools-5.3.3-py3-none-any.whl", hash = "sha256:0abad1021d3f8325b2fc1d2e9c8b9c9d57b04c3932657a72465447332c24d945"}, + {file = "cachetools-5.3.3.tar.gz", hash = "sha256:ba29e2dfa0b8b556606f097407ed1aa62080ee108ab0dc5ec9d6a723a007d105"}, ] [[package]] @@ -388,13 +388,13 @@ files = [ [[package]] name = "coolname" -version = "1.1.0" +version = "2.2.0" description = "Random name and slug generator" optional = false python-versions = "*" files = [ - {file = "coolname-1.1.0-py2.py3-none-any.whl", hash = "sha256:e6a83a0ac88640f4f3d2070438dbe112fe80cfebc119c93bd402976ec84c0978"}, - {file = "coolname-1.1.0.tar.gz", hash = "sha256:410fe6ea9999bf96f2856ef0c726d5f38782bbefb7bb1aca0e91e0dc98ed09e3"}, + {file = "coolname-2.2.0-py2.py3-none-any.whl", hash = "sha256:4d1563186cfaf71b394d5df4c744f8c41303b6846413645e31d31915cdeb13e8"}, + {file = "coolname-2.2.0.tar.gz", hash = "sha256:6c5d5731759104479e7ca195a9b64f7900ac5bead40183c09323c7d0be9e75c7"}, ] [[package]] @@ -662,13 +662,13 @@ google-crc32c = "1.3.0" [[package]] name = "google-api-core" -version = "2.16.2" +version = "2.17.1" description = "Google API client core library" optional = false python-versions = ">=3.7" files = [ - {file = "google-api-core-2.16.2.tar.gz", hash = "sha256:032d37b45d1d6bdaf68fb11ff621e2593263a239fa9246e2e94325f9c47876d2"}, - {file = "google_api_core-2.16.2-py3-none-any.whl", hash = "sha256:449ca0e3f14c179b4165b664256066c7861610f70b6ffe54bb01a04e9b466929"}, + {file = "google-api-core-2.17.1.tar.gz", hash = "sha256:9df18a1f87ee0df0bc4eea2770ebc4228392d8cc4066655b320e2cfccb15db95"}, + {file = "google_api_core-2.17.1-py3-none-any.whl", hash = "sha256:610c5b90092c360736baccf17bd3efbcb30dd380e7a6dc28a71059edb8bd0d8e"}, ] [package.dependencies] @@ -692,13 +692,13 @@ grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] [[package]] name = "google-auth" -version = "2.27.0" +version = "2.28.1" description = "Google Authentication Library" optional = false python-versions = ">=3.7" files = [ - {file = "google-auth-2.27.0.tar.gz", hash = "sha256:e863a56ccc2d8efa83df7a80272601e43487fa9a728a376205c86c26aaefa821"}, - {file = "google_auth-2.27.0-py2.py3-none-any.whl", hash = "sha256:8e4bad367015430ff253fe49d500fdc3396c1a434db5740828c728e45bcce245"}, + {file = "google-auth-2.28.1.tar.gz", hash = "sha256:34fc3046c257cedcf1622fc4b31fc2be7923d9b4d44973d481125ecc50d83885"}, + {file = "google_auth-2.28.1-py2.py3-none-any.whl", hash = "sha256:25141e2d7a14bfcba945f5e9827f98092716e99482562f15306e5b026e21aa72"}, ] [package.dependencies] @@ -764,17 +764,18 @@ grpc = ["grpcio (>=1.38.0,<2.0dev)", "grpcio-status (>=1.38.0,<2.0.dev0)"] [[package]] name = "google-cloud-pubsub" -version = "2.19.1" +version = "2.20.0" description = "Google Cloud Pub/Sub API client library" optional = false python-versions = ">=3.7" files = [ - {file = "google-cloud-pubsub-2.19.1.tar.gz", hash = "sha256:c10d95ebaf903f923b14aa8ec5b7c80916138edf299c65a1c1a9431fd5665d25"}, - {file = "google_cloud_pubsub-2.19.1-py2.py3-none-any.whl", hash = "sha256:602eacbe5735f38d3ae384ece8f235db0594de78ad718a35fcb948dc4964612c"}, + {file = "google-cloud-pubsub-2.20.0.tar.gz", hash = "sha256:48c8e17a8168c43e3188635cbd9e07fbe3004120433712ce84b3a04bbf18c188"}, + {file = "google_cloud_pubsub-2.20.0-py2.py3-none-any.whl", hash = "sha256:8c69ed04800f4f552cdf3b9028f06d9271ac6e60443b2308c984def442e69684"}, ] [package.dependencies] google-api-core = {version = ">=1.34.0,<2.0.dev0 || >=2.11.dev0,<3.0.0dev", extras = ["grpc"]} +google-auth = ">=2.14.1,<3.0.0dev" grpc-google-iam-v1 = ">=0.12.4,<1.0.0dev" grpcio = ">=1.51.3,<2.0dev" grpcio-status = ">=1.33.2" @@ -789,35 +790,36 @@ libcst = ["libcst (>=0.3.10)"] [[package]] name = "google-cloud-secret-manager" -version = "2.18.0" +version = "2.18.3" description = "Google Cloud Secret Manager API client library" optional = false python-versions = ">=3.7" files = [ - {file = "google-cloud-secret-manager-2.18.0.tar.gz", hash = "sha256:4e499bd33ff7aeff271bd67487e21d5404297a86dc4873ee861d637b01b30b4e"}, - {file = "google_cloud_secret_manager-2.18.0-py2.py3-none-any.whl", hash = "sha256:06ce001648a75fc7e992e3e658772b10a6ebc738a49463204c44409137ad5890"}, + {file = "google-cloud-secret-manager-2.18.3.tar.gz", hash = "sha256:1db2f409324536e34f985081d389e3974ca3a3668df7845cad0be03ab8c0fa7d"}, + {file = "google_cloud_secret_manager-2.18.3-py2.py3-none-any.whl", hash = "sha256:4d4af82bddd9099ebdbe79e0c6b68f6c6cabea8323a3c1275bcead8f56310fb7"}, ] [package.dependencies] -google-api-core = {version = ">=1.34.0,<2.0.dev0 || >=2.11.dev0,<3.0.0dev", extras = ["grpc"]} +google-api-core = {version = ">=1.34.1,<2.0.dev0 || >=2.11.dev0,<3.0.0dev", extras = ["grpc"]} +google-auth = ">=2.14.1,<2.24.0 || >2.24.0,<2.25.0 || >2.25.0,<3.0.0dev" grpc-google-iam-v1 = ">=0.12.4,<1.0.0dev" proto-plus = ">=1.22.3,<2.0.0dev" protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev" [[package]] name = "google-cloud-storage" -version = "2.14.0" +version = "2.15.0" description = "Google Cloud Storage API client library" optional = false python-versions = ">=3.7" files = [ - {file = "google-cloud-storage-2.14.0.tar.gz", hash = "sha256:2d23fcf59b55e7b45336729c148bb1c464468c69d5efbaee30f7201dd90eb97e"}, - {file = "google_cloud_storage-2.14.0-py2.py3-none-any.whl", hash = "sha256:8641243bbf2a2042c16a6399551fbb13f062cbc9a2de38d6c0bb5426962e9dbd"}, + {file = "google-cloud-storage-2.15.0.tar.gz", hash = "sha256:7560a3c48a03d66c553dc55215d35883c680fe0ab44c23aa4832800ccc855c74"}, + {file = "google_cloud_storage-2.15.0-py2.py3-none-any.whl", hash = "sha256:5d9237f88b648e1d724a0f20b5cde65996a37fe51d75d17660b1404097327dd2"}, ] [package.dependencies] -google-api-core = ">=1.31.5,<2.0.dev0 || >2.3.0,<3.0.0dev" -google-auth = ">=2.23.3,<3.0dev" +google-api-core = ">=2.15.0,<3.0.0dev" +google-auth = ">=2.26.1,<3.0dev" google-cloud-core = ">=2.3.0,<3.0dev" google-crc32c = ">=1.0,<2.0dev" google-resumable-media = ">=2.6.0" @@ -935,84 +937,84 @@ protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4 [[package]] name = "grpcio" -version = "1.60.1" +version = "1.62.0" description = "HTTP/2-based RPC framework" optional = false python-versions = ">=3.7" files = [ - {file = "grpcio-1.60.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:14e8f2c84c0832773fb3958240c69def72357bc11392571f87b2d7b91e0bb092"}, - {file = "grpcio-1.60.1-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:33aed0a431f5befeffd9d346b0fa44b2c01aa4aeae5ea5b2c03d3e25e0071216"}, - {file = "grpcio-1.60.1-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:fead980fbc68512dfd4e0c7b1f5754c2a8e5015a04dea454b9cada54a8423525"}, - {file = "grpcio-1.60.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:082081e6a36b6eb5cf0fd9a897fe777dbb3802176ffd08e3ec6567edd85bc104"}, - {file = "grpcio-1.60.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:55ccb7db5a665079d68b5c7c86359ebd5ebf31a19bc1a91c982fd622f1e31ff2"}, - {file = "grpcio-1.60.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:9b54577032d4f235452f77a83169b6527bf4b77d73aeada97d45b2aaf1bf5ce0"}, - {file = "grpcio-1.60.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7d142bcd604166417929b071cd396aa13c565749a4c840d6c702727a59d835eb"}, - {file = "grpcio-1.60.1-cp310-cp310-win32.whl", hash = "sha256:2a6087f234cb570008a6041c8ffd1b7d657b397fdd6d26e83d72283dae3527b1"}, - {file = "grpcio-1.60.1-cp310-cp310-win_amd64.whl", hash = "sha256:f2212796593ad1d0235068c79836861f2201fc7137a99aa2fea7beeb3b101177"}, - {file = "grpcio-1.60.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:79ae0dc785504cb1e1788758c588c711f4e4a0195d70dff53db203c95a0bd303"}, - {file = "grpcio-1.60.1-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:4eec8b8c1c2c9b7125508ff7c89d5701bf933c99d3910e446ed531cd16ad5d87"}, - {file = "grpcio-1.60.1-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:8c9554ca8e26241dabe7951aa1fa03a1ba0856688ecd7e7bdbdd286ebc272e4c"}, - {file = "grpcio-1.60.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:91422ba785a8e7a18725b1dc40fbd88f08a5bb4c7f1b3e8739cab24b04fa8a03"}, - {file = "grpcio-1.60.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cba6209c96828711cb7c8fcb45ecef8c8859238baf15119daa1bef0f6c84bfe7"}, - {file = "grpcio-1.60.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c71be3f86d67d8d1311c6076a4ba3b75ba5703c0b856b4e691c9097f9b1e8bd2"}, - {file = "grpcio-1.60.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:af5ef6cfaf0d023c00002ba25d0751e5995fa0e4c9eec6cd263c30352662cbce"}, - {file = "grpcio-1.60.1-cp311-cp311-win32.whl", hash = "sha256:a09506eb48fa5493c58f946c46754ef22f3ec0df64f2b5149373ff31fb67f3dd"}, - {file = "grpcio-1.60.1-cp311-cp311-win_amd64.whl", hash = "sha256:49c9b6a510e3ed8df5f6f4f3c34d7fbf2d2cae048ee90a45cd7415abab72912c"}, - {file = "grpcio-1.60.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:b58b855d0071575ea9c7bc0d84a06d2edfbfccec52e9657864386381a7ce1ae9"}, - {file = "grpcio-1.60.1-cp312-cp312-macosx_10_10_universal2.whl", hash = "sha256:a731ac5cffc34dac62053e0da90f0c0b8560396a19f69d9703e88240c8f05858"}, - {file = "grpcio-1.60.1-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:cf77f8cf2a651fbd869fbdcb4a1931464189cd210abc4cfad357f1cacc8642a6"}, - {file = "grpcio-1.60.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c557e94e91a983e5b1e9c60076a8fd79fea1e7e06848eb2e48d0ccfb30f6e073"}, - {file = "grpcio-1.60.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:069fe2aeee02dfd2135d562d0663fe70fbb69d5eed6eb3389042a7e963b54de8"}, - {file = "grpcio-1.60.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:cb0af13433dbbd1c806e671d81ec75bd324af6ef75171fd7815ca3074fe32bfe"}, - {file = "grpcio-1.60.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2f44c32aef186bbba254129cea1df08a20be414144ac3bdf0e84b24e3f3b2e05"}, - {file = "grpcio-1.60.1-cp312-cp312-win32.whl", hash = "sha256:a212e5dea1a4182e40cd3e4067ee46be9d10418092ce3627475e995cca95de21"}, - {file = "grpcio-1.60.1-cp312-cp312-win_amd64.whl", hash = "sha256:6e490fa5f7f5326222cb9f0b78f207a2b218a14edf39602e083d5f617354306f"}, - {file = "grpcio-1.60.1-cp37-cp37m-linux_armv7l.whl", hash = "sha256:4216e67ad9a4769117433814956031cb300f85edc855252a645a9a724b3b6594"}, - {file = "grpcio-1.60.1-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:73e14acd3d4247169955fae8fb103a2b900cfad21d0c35f0dcd0fdd54cd60367"}, - {file = "grpcio-1.60.1-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:6ecf21d20d02d1733e9c820fb5c114c749d888704a7ec824b545c12e78734d1c"}, - {file = "grpcio-1.60.1-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:33bdea30dcfd4f87b045d404388469eb48a48c33a6195a043d116ed1b9a0196c"}, - {file = "grpcio-1.60.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53b69e79d00f78c81eecfb38f4516080dc7f36a198b6b37b928f1c13b3c063e9"}, - {file = "grpcio-1.60.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:39aa848794b887120b1d35b1b994e445cc028ff602ef267f87c38122c1add50d"}, - {file = "grpcio-1.60.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:72153a0d2e425f45b884540a61c6639436ddafa1829a42056aa5764b84108b8e"}, - {file = "grpcio-1.60.1-cp37-cp37m-win_amd64.whl", hash = "sha256:50d56280b482875d1f9128ce596e59031a226a8b84bec88cb2bf76c289f5d0de"}, - {file = "grpcio-1.60.1-cp38-cp38-linux_armv7l.whl", hash = "sha256:6d140bdeb26cad8b93c1455fa00573c05592793c32053d6e0016ce05ba267549"}, - {file = "grpcio-1.60.1-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:bc808924470643b82b14fe121923c30ec211d8c693e747eba8a7414bc4351a23"}, - {file = "grpcio-1.60.1-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:70c83bb530572917be20c21f3b6be92cd86b9aecb44b0c18b1d3b2cc3ae47df0"}, - {file = "grpcio-1.60.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9b106bc52e7f28170e624ba61cc7dc6829566e535a6ec68528f8e1afbed1c41f"}, - {file = "grpcio-1.60.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:30e980cd6db1088c144b92fe376747328d5554bc7960ce583ec7b7d81cd47287"}, - {file = "grpcio-1.60.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:0c5807e9152eff15f1d48f6b9ad3749196f79a4a050469d99eecb679be592acc"}, - {file = "grpcio-1.60.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f1c3dc536b3ee124e8b24feb7533e5c70b9f2ef833e3b2e5513b2897fd46763a"}, - {file = "grpcio-1.60.1-cp38-cp38-win32.whl", hash = "sha256:d7404cebcdb11bb5bd40bf94131faf7e9a7c10a6c60358580fe83913f360f929"}, - {file = "grpcio-1.60.1-cp38-cp38-win_amd64.whl", hash = "sha256:c8754c75f55781515a3005063d9a05878b2cfb3cb7e41d5401ad0cf19de14872"}, - {file = "grpcio-1.60.1-cp39-cp39-linux_armv7l.whl", hash = "sha256:0250a7a70b14000fa311de04b169cc7480be6c1a769b190769d347939d3232a8"}, - {file = "grpcio-1.60.1-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:660fc6b9c2a9ea3bb2a7e64ba878c98339abaf1811edca904ac85e9e662f1d73"}, - {file = "grpcio-1.60.1-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:76eaaba891083fcbe167aa0f03363311a9f12da975b025d30e94b93ac7a765fc"}, - {file = "grpcio-1.60.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e5d97c65ea7e097056f3d1ead77040ebc236feaf7f71489383d20f3b4c28412a"}, - {file = "grpcio-1.60.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bb2a2911b028f01c8c64d126f6b632fcd8a9ac975aa1b3855766c94e4107180"}, - {file = "grpcio-1.60.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:5a1ebbae7e2214f51b1f23b57bf98eeed2cf1ba84e4d523c48c36d5b2f8829ff"}, - {file = "grpcio-1.60.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:9a66f4d2a005bc78e61d805ed95dedfcb35efa84b7bba0403c6d60d13a3de2d6"}, - {file = "grpcio-1.60.1-cp39-cp39-win32.whl", hash = "sha256:8d488fbdbf04283f0d20742b64968d44825617aa6717b07c006168ed16488804"}, - {file = "grpcio-1.60.1-cp39-cp39-win_amd64.whl", hash = "sha256:61b7199cd2a55e62e45bfb629a35b71fc2c0cb88f686a047f25b1112d3810904"}, - {file = "grpcio-1.60.1.tar.gz", hash = "sha256:dd1d3a8d1d2e50ad9b59e10aa7f07c7d1be2b367f3f2d33c5fade96ed5460962"}, + {file = "grpcio-1.62.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:136ffd79791b1eddda8d827b607a6285474ff8a1a5735c4947b58c481e5e4271"}, + {file = "grpcio-1.62.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:d6a56ba703be6b6267bf19423d888600c3f574ac7c2cc5e6220af90662a4d6b0"}, + {file = "grpcio-1.62.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:4cd356211579043fce9f52acc861e519316fff93980a212c8109cca8f47366b6"}, + {file = "grpcio-1.62.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e803e9b58d8f9b4ff0ea991611a8d51b31c68d2e24572cd1fe85e99e8cc1b4f8"}, + {file = "grpcio-1.62.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f4c04fe33039b35b97c02d2901a164bbbb2f21fb9c4e2a45a959f0b044c3512c"}, + {file = "grpcio-1.62.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:95370c71b8c9062f9ea033a0867c4c73d6f0ff35113ebd2618171ec1f1e903e0"}, + {file = "grpcio-1.62.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c912688acc05e4ff012c8891803659d6a8a8b5106f0f66e0aed3fb7e77898fa6"}, + {file = "grpcio-1.62.0-cp310-cp310-win32.whl", hash = "sha256:821a44bd63d0f04e33cf4ddf33c14cae176346486b0df08b41a6132b976de5fc"}, + {file = "grpcio-1.62.0-cp310-cp310-win_amd64.whl", hash = "sha256:81531632f93fece32b2762247c4c169021177e58e725494f9a746ca62c83acaa"}, + {file = "grpcio-1.62.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:3fa15850a6aba230eed06b236287c50d65a98f05054a0f01ccedf8e1cc89d57f"}, + {file = "grpcio-1.62.0-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:36df33080cd7897623feff57831eb83c98b84640b016ce443305977fac7566fb"}, + {file = "grpcio-1.62.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:7a195531828b46ea9c4623c47e1dc45650fc7206f8a71825898dd4c9004b0928"}, + {file = "grpcio-1.62.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ab140a3542bbcea37162bdfc12ce0d47a3cda3f2d91b752a124cc9fe6776a9e2"}, + {file = "grpcio-1.62.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f9d6c3223914abb51ac564dc9c3782d23ca445d2864321b9059d62d47144021"}, + {file = "grpcio-1.62.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:fbe0c20ce9a1cff75cfb828b21f08d0a1ca527b67f2443174af6626798a754a4"}, + {file = "grpcio-1.62.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:38f69de9c28c1e7a8fd24e4af4264726637b72f27c2099eaea6e513e7142b47e"}, + {file = "grpcio-1.62.0-cp311-cp311-win32.whl", hash = "sha256:ce1aafdf8d3f58cb67664f42a617af0e34555fe955450d42c19e4a6ad41c84bd"}, + {file = "grpcio-1.62.0-cp311-cp311-win_amd64.whl", hash = "sha256:eef1d16ac26c5325e7d39f5452ea98d6988c700c427c52cbc7ce3201e6d93334"}, + {file = "grpcio-1.62.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:8aab8f90b2a41208c0a071ec39a6e5dbba16fd827455aaa070fec241624ccef8"}, + {file = "grpcio-1.62.0-cp312-cp312-macosx_10_10_universal2.whl", hash = "sha256:62aa1659d8b6aad7329ede5d5b077e3d71bf488d85795db517118c390358d5f6"}, + {file = "grpcio-1.62.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:0d7ae7fc7dbbf2d78d6323641ded767d9ec6d121aaf931ec4a5c50797b886532"}, + {file = "grpcio-1.62.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f359d635ee9428f0294bea062bb60c478a8ddc44b0b6f8e1f42997e5dc12e2ee"}, + {file = "grpcio-1.62.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77d48e5b1f8f4204889f1acf30bb57c30378e17c8d20df5acbe8029e985f735c"}, + {file = "grpcio-1.62.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:662d3df5314ecde3184cf87ddd2c3a66095b3acbb2d57a8cada571747af03873"}, + {file = "grpcio-1.62.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:92cdb616be44c8ac23a57cce0243af0137a10aa82234f23cd46e69e115071388"}, + {file = "grpcio-1.62.0-cp312-cp312-win32.whl", hash = "sha256:0b9179478b09ee22f4a36b40ca87ad43376acdccc816ce7c2193a9061bf35701"}, + {file = "grpcio-1.62.0-cp312-cp312-win_amd64.whl", hash = "sha256:614c3ed234208e76991992342bab725f379cc81c7dd5035ee1de2f7e3f7a9842"}, + {file = "grpcio-1.62.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:7e1f51e2a460b7394670fdb615e26d31d3260015154ea4f1501a45047abe06c9"}, + {file = "grpcio-1.62.0-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:bcff647e7fe25495e7719f779cc219bbb90b9e79fbd1ce5bda6aae2567f469f2"}, + {file = "grpcio-1.62.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:56ca7ba0b51ed0de1646f1735154143dcbdf9ec2dbe8cc6645def299bb527ca1"}, + {file = "grpcio-1.62.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e84bfb2a734e4a234b116be208d6f0214e68dcf7804306f97962f93c22a1839"}, + {file = "grpcio-1.62.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c1488b31a521fbba50ae86423f5306668d6f3a46d124f7819c603979fc538c4"}, + {file = "grpcio-1.62.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:98d8f4eb91f1ce0735bf0b67c3b2a4fea68b52b2fd13dc4318583181f9219b4b"}, + {file = "grpcio-1.62.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:b3d3d755cfa331d6090e13aac276d4a3fb828bf935449dc16c3d554bf366136b"}, + {file = "grpcio-1.62.0-cp37-cp37m-win_amd64.whl", hash = "sha256:a33f2bfd8a58a02aab93f94f6c61279be0f48f99fcca20ebaee67576cd57307b"}, + {file = "grpcio-1.62.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:5e709f7c8028ce0443bddc290fb9c967c1e0e9159ef7a030e8c21cac1feabd35"}, + {file = "grpcio-1.62.0-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:2f3d9a4d0abb57e5f49ed5039d3ed375826c2635751ab89dcc25932ff683bbb6"}, + {file = "grpcio-1.62.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:62ccb92f594d3d9fcd00064b149a0187c246b11e46ff1b7935191f169227f04c"}, + {file = "grpcio-1.62.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:921148f57c2e4b076af59a815467d399b7447f6e0ee10ef6d2601eb1e9c7f402"}, + {file = "grpcio-1.62.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f897b16190b46bc4d4aaf0a32a4b819d559a37a756d7c6b571e9562c360eed72"}, + {file = "grpcio-1.62.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1bc8449084fe395575ed24809752e1dc4592bb70900a03ca42bf236ed5bf008f"}, + {file = "grpcio-1.62.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:81d444e5e182be4c7856cd33a610154fe9ea1726bd071d07e7ba13fafd202e38"}, + {file = "grpcio-1.62.0-cp38-cp38-win32.whl", hash = "sha256:88f41f33da3840b4a9bbec68079096d4caf629e2c6ed3a72112159d570d98ebe"}, + {file = "grpcio-1.62.0-cp38-cp38-win_amd64.whl", hash = "sha256:fc2836cb829895ee190813446dce63df67e6ed7b9bf76060262c55fcd097d270"}, + {file = "grpcio-1.62.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:fcc98cff4084467839d0a20d16abc2a76005f3d1b38062464d088c07f500d170"}, + {file = "grpcio-1.62.0-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:0d3dee701e48ee76b7d6fbbba18ba8bc142e5b231ef7d3d97065204702224e0e"}, + {file = "grpcio-1.62.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:b7a6be562dd18e5d5bec146ae9537f20ae1253beb971c0164f1e8a2f5a27e829"}, + {file = "grpcio-1.62.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:29cb592c4ce64a023712875368bcae13938c7f03e99f080407e20ffe0a9aa33b"}, + {file = "grpcio-1.62.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1eda79574aec8ec4d00768dcb07daba60ed08ef32583b62b90bbf274b3c279f7"}, + {file = "grpcio-1.62.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7eea57444a354ee217fda23f4b479a4cdfea35fb918ca0d8a0e73c271e52c09c"}, + {file = "grpcio-1.62.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0e97f37a3b7c89f9125b92d22e9c8323f4e76e7993ba7049b9f4ccbe8bae958a"}, + {file = "grpcio-1.62.0-cp39-cp39-win32.whl", hash = "sha256:39cd45bd82a2e510e591ca2ddbe22352e8413378852ae814549c162cf3992a93"}, + {file = "grpcio-1.62.0-cp39-cp39-win_amd64.whl", hash = "sha256:b71c65427bf0ec6a8b48c68c17356cb9fbfc96b1130d20a07cb462f4e4dcdcd5"}, + {file = "grpcio-1.62.0.tar.gz", hash = "sha256:748496af9238ac78dcd98cce65421f1adce28c3979393e3609683fcd7f3880d7"}, ] [package.extras] -protobuf = ["grpcio-tools (>=1.60.1)"] +protobuf = ["grpcio-tools (>=1.62.0)"] [[package]] name = "grpcio-status" -version = "1.60.1" +version = "1.62.0" description = "Status proto mapping for gRPC" optional = false python-versions = ">=3.6" files = [ - {file = "grpcio-status-1.60.1.tar.gz", hash = "sha256:61b5aab8989498e8aa142c20b88829ea5d90d18c18c853b9f9e6d407d37bf8b4"}, - {file = "grpcio_status-1.60.1-py3-none-any.whl", hash = "sha256:3034fdb239185b6e0f3169d08c268c4507481e4b8a434c21311a03d9eb5889a0"}, + {file = "grpcio-status-1.62.0.tar.gz", hash = "sha256:0d693e9c09880daeaac060d0c3dba1ae470a43c99e5d20dfeafd62cf7e08a85d"}, + {file = "grpcio_status-1.62.0-py3-none-any.whl", hash = "sha256:3baac03fcd737310e67758c4082a188107f771d32855bce203331cd4c9aa687a"}, ] [package.dependencies] googleapis-common-protos = ">=1.5.5" -grpcio = ">=1.60.1" +grpcio = ">=1.62.0" protobuf = ">=4.21.6" [[package]] @@ -1782,13 +1784,13 @@ testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "no [[package]] name = "python-dateutil" -version = "2.8.2" +version = "2.9.0.post0" description = "Extensions to the standard Python datetime module" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ - {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, - {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, + {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, + {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, ] [package.dependencies] @@ -1998,13 +2000,13 @@ pyasn1 = ">=0.1.3" [[package]] name = "ruamel-yaml" -version = "0.18.5" +version = "0.18.6" description = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order" optional = false python-versions = ">=3.7" files = [ - {file = "ruamel.yaml-0.18.5-py3-none-any.whl", hash = "sha256:a013ac02f99a69cdd6277d9664689eb1acba07069f912823177c5eced21a6ada"}, - {file = "ruamel.yaml-0.18.5.tar.gz", hash = "sha256:61917e3a35a569c1133a8f772e1226961bf5a1198bea7e23f06a0841dea1ab0e"}, + {file = "ruamel.yaml-0.18.6-py3-none-any.whl", hash = "sha256:57b53ba33def16c4f3d807c0ccbc00f8a6081827e81ba2491691b76882d0c636"}, + {file = "ruamel.yaml-0.18.6.tar.gz", hash = "sha256:8b27e6a217e786c6fbe5634d8f3f11bc63e0f80f6a5890f28863d9c45aac311b"}, ] [package.dependencies] @@ -2535,13 +2537,13 @@ typing-extensions = ">=3.7.4" [[package]] name = "tzdata" -version = "2023.4" +version = "2024.1" description = "Provider of IANA time zone data" optional = false python-versions = ">=2" files = [ - {file = "tzdata-2023.4-py2.py3-none-any.whl", hash = "sha256:aa3ace4329eeacda5b7beb7ea08ece826c28d761cda36e747cfbf97996d39bf3"}, - {file = "tzdata-2023.4.tar.gz", hash = "sha256:dd54c94f294765522c77399649b4fefd95522479a664a0cec87f41bebc6148c9"}, + {file = "tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"}, + {file = "tzdata-2024.1.tar.gz", hash = "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd"}, ] [[package]] @@ -2581,13 +2583,13 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "virtualenv" -version = "20.25.0" +version = "20.25.1" description = "Virtual Python Environment builder" optional = false python-versions = ">=3.7" files = [ - {file = "virtualenv-20.25.0-py3-none-any.whl", hash = "sha256:4238949c5ffe6876362d9c0180fc6c3a824a7b12b80604eeb8085f2ed7460de3"}, - {file = "virtualenv-20.25.0.tar.gz", hash = "sha256:bf51c0d9c7dd63ea8e44086fa1e4fb1093a31e963b86959257378aef020e1f1b"}, + {file = "virtualenv-20.25.1-py3-none-any.whl", hash = "sha256:961c026ac520bac5f69acb8ea063e8a4f071bcc9457b9c1f28f6b085c511583a"}, + {file = "virtualenv-20.25.1.tar.gz", hash = "sha256:e08e13ecdca7a0bd53798f356d5831434afa5b07b93f0abdf0797b7a06ffe197"}, ] [package.dependencies] @@ -2649,4 +2651,4 @@ hdf5 = ["h5py"] [metadata] lock-version = "2.0" python-versions = "^3.7.1" -content-hash = "ccaa2540a6d43f5c61d5fff4e770a526673a8e64892c8668e3a0b6339128e3ed" +content-hash = "d229df7438e9142fa696a9db7dfd2969b21afbfc1dbc541d4cd10104d6ad7ae8" diff --git a/pyproject.toml b/pyproject.toml index aaeddbfc8..c9a957d8c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ classifiers = [ [tool.poetry.dependencies] python = "^3.7.1" click = ">=7,<9" -coolname = "^1.1" +coolname = "^2" Flask = "^2" google-auth = ">=1.27.0,<3" google-cloud-pubsub = "^2.5" From 0ee05846a3f705c56413a38cbfb1dd5df90208f3 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 6 Mar 2024 12:02:24 +0000 Subject: [PATCH 015/169] OPS: Add bigquery IAM roles to terraform config --- terraform/iam.tf | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/terraform/iam.tf b/terraform/iam.tf index 8d5ad51f6..4ae3efa0a 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -91,6 +91,33 @@ resource "google_project_iam_binding" "bigquery_dataeditor" { } +resource "google_project_iam_binding" "bigquery_dataviewer" { + project = var.project + role = "roles/bigquery.dataViewer" + members = [ + "serviceAccount:${google_service_account.dev_cortadocodes_service_account.email}", + ] +} + + +resource "google_project_iam_binding" "bigquery_jobuser" { + project = var.project + role = "roles/bigquery.jobUser" + members = [ + "serviceAccount:${google_service_account.dev_cortadocodes_service_account.email}", + ] +} + + +resource "google_project_iam_binding" "bigquery_readsessionuser" { + project = var.project + role = "roles/bigquery.readSessionUser" + members = [ + "serviceAccount:${google_service_account.dev_cortadocodes_service_account.email}", + ] +} + + resource "google_iam_workload_identity_pool" "github_actions_pool" { display_name = "github-actions-pool" project = var.project From 63be56a3c37284d9b82bc46a400f5cf981d97b13 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 6 Mar 2024 12:02:48 +0000 Subject: [PATCH 016/169] DEP: Add `db-dtypes` for converting bigquery rows to dataframes skipci --- poetry.lock | 56 +++++++++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 1 + 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index fe52a6400..e91eec3fe 100644 --- a/poetry.lock +++ b/poetry.lock @@ -501,6 +501,23 @@ calendars = ["convertdate", "convertdate", "hijri-converter"] fasttext = ["fasttext"] langdetect = ["langdetect"] +[[package]] +name = "db-dtypes" +version = "1.2.0" +description = "Pandas Data Types for SQL systems (BigQuery, Spanner)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "db-dtypes-1.2.0.tar.gz", hash = "sha256:3531bb1fb8b5fbab33121fe243ccc2ade16ab2524f4c113b05cc702a1908e6ea"}, + {file = "db_dtypes-1.2.0-py2.py3-none-any.whl", hash = "sha256:6320bddd31d096447ef749224d64aab00972ed20e4392d86f7d8b81ad79f7ff0"}, +] + +[package.dependencies] +numpy = ">=1.16.6" +packaging = ">=17.0" +pandas = ">=0.24.2" +pyarrow = ">=3.0.0" + [[package]] name = "dict2css" version = "0.3.0.post1" @@ -1661,6 +1678,43 @@ files = [ {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, ] +[[package]] +name = "pyarrow" +version = "12.0.1" +description = "Python library for Apache Arrow" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pyarrow-12.0.1-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:6d288029a94a9bb5407ceebdd7110ba398a00412c5b0155ee9813a40d246c5df"}, + {file = "pyarrow-12.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:345e1828efdbd9aa4d4de7d5676778aba384a2c3add896d995b23d368e60e5af"}, + {file = "pyarrow-12.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8d6009fdf8986332b2169314da482baed47ac053311c8934ac6651e614deacd6"}, + {file = "pyarrow-12.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d3c4cbbf81e6dd23fe921bc91dc4619ea3b79bc58ef10bce0f49bdafb103daf"}, + {file = "pyarrow-12.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:cdacf515ec276709ac8042c7d9bd5be83b4f5f39c6c037a17a60d7ebfd92c890"}, + {file = "pyarrow-12.0.1-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:749be7fd2ff260683f9cc739cb862fb11be376de965a2a8ccbf2693b098db6c7"}, + {file = "pyarrow-12.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6895b5fb74289d055c43db3af0de6e16b07586c45763cb5e558d38b86a91e3a7"}, + {file = "pyarrow-12.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1887bdae17ec3b4c046fcf19951e71b6a619f39fa674f9881216173566c8f718"}, + {file = "pyarrow-12.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e2c9cb8eeabbadf5fcfc3d1ddea616c7ce893db2ce4dcef0ac13b099ad7ca082"}, + {file = "pyarrow-12.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:ce4aebdf412bd0eeb800d8e47db854f9f9f7e2f5a0220440acf219ddfddd4f63"}, + {file = "pyarrow-12.0.1-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:e0d8730c7f6e893f6db5d5b86eda42c0a130842d101992b581e2138e4d5663d3"}, + {file = "pyarrow-12.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:43364daec02f69fec89d2315f7fbfbeec956e0d991cbbef471681bd77875c40f"}, + {file = "pyarrow-12.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:051f9f5ccf585f12d7de836e50965b3c235542cc896959320d9776ab93f3b33d"}, + {file = "pyarrow-12.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:be2757e9275875d2a9c6e6052ac7957fbbfc7bc7370e4a036a9b893e96fedaba"}, + {file = "pyarrow-12.0.1-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:cf812306d66f40f69e684300f7af5111c11f6e0d89d6b733e05a3de44961529d"}, + {file = "pyarrow-12.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:459a1c0ed2d68671188b2118c63bac91eaef6fc150c77ddd8a583e3c795737bf"}, + {file = "pyarrow-12.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:85e705e33eaf666bbe508a16fd5ba27ca061e177916b7a317ba5a51bee43384c"}, + {file = "pyarrow-12.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9120c3eb2b1f6f516a3b7a9714ed860882d9ef98c4b17edcdc91d95b7528db60"}, + {file = "pyarrow-12.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:c780f4dc40460015d80fcd6a6140de80b615349ed68ef9adb653fe351778c9b3"}, + {file = "pyarrow-12.0.1-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:a3c63124fc26bf5f95f508f5d04e1ece8cc23a8b0af2a1e6ab2b1ec3fdc91b24"}, + {file = "pyarrow-12.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b13329f79fa4472324f8d32dc1b1216616d09bd1e77cfb13104dec5463632c36"}, + {file = "pyarrow-12.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb656150d3d12ec1396f6dde542db1675a95c0cc8366d507347b0beed96e87ca"}, + {file = "pyarrow-12.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6251e38470da97a5b2e00de5c6a049149f7b2bd62f12fa5dbb9ac674119ba71a"}, + {file = "pyarrow-12.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:3de26da901216149ce086920547dfff5cd22818c9eab67ebc41e863a5883bac7"}, + {file = "pyarrow-12.0.1.tar.gz", hash = "sha256:cce317fc96e5b71107bf1f9f184d5e54e2bd14bbf3f9a3d62819961f0af86fec"}, +] + +[package.dependencies] +numpy = ">=1.16.6" + [[package]] name = "pyasn1" version = "0.5.1" @@ -2651,4 +2705,4 @@ hdf5 = ["h5py"] [metadata] lock-version = "2.0" python-versions = "^3.7.1" -content-hash = "d229df7438e9142fa696a9db7dfd2969b21afbfc1dbc541d4cd10104d6ad7ae8" +content-hash = "1572a843b2a77b5376349ff6cd212d39644931ab91b8e7b61c12709eda55617e" diff --git a/pyproject.toml b/pyproject.toml index c9a957d8c..1d24ee7d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,7 @@ h5py = { version = "^3.6", optional = true } twined = "^0.5.1" packaging = ">=20.4" google-cloud-bigquery = "^3.18.0" +db-dtypes = "^1.2.0" [tool.poetry.extras] hdf5 = ["h5py"] From a817f327df9ff88353037f298fd0624b8d9e55e1 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 6 Mar 2024 13:25:14 +0000 Subject: [PATCH 017/169] FEA: Add function for getting events from BigQuery --- octue/cloud/pub_sub/bigquery.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 octue/cloud/pub_sub/bigquery.py diff --git a/octue/cloud/pub_sub/bigquery.py b/octue/cloud/pub_sub/bigquery.py new file mode 100644 index 000000000..8df3678e5 --- /dev/null +++ b/octue/cloud/pub_sub/bigquery.py @@ -0,0 +1,32 @@ +from google.cloud import bigquery + + +def get_events(table_id, question_uuid, limit=1000): + """Get Octue service events for a question from a Google BigQuery table. + + :param str table_id: the full ID of the table e.g. "your-project.your-dataset.your-table" + :param str question_uuid: the UUID of the question to get the events for + :param int limit: the maximum number of events to return. + :return list(dict): the events for the question + """ + client = bigquery.Client() + + query = f""" + SELECT * FROM `{table_id}` + WHERE CONTAINS_SUBSTR(subscription_name, @question_uuid) + ORDER BY `publish_time` + LIMIT @limit + """ + + job_config = bigquery.QueryJobConfig( + query_parameters=[ + bigquery.ScalarQueryParameter("question_uuid", "STRING", question_uuid), + bigquery.ScalarQueryParameter("limit", "INTEGER", limit), + ] + ) + + query_job = client.query(query, job_config=job_config) + rows = query_job.result() + messages = rows.to_dataframe() + + return messages.to_dict(orient="records") From 639102e0c07d7ef40fb4095b269ffaf41d16b607 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 6 Mar 2024 13:31:21 +0000 Subject: [PATCH 018/169] FIX: Fix `api_access_endpoint` usage in `mock_generate_signed_url` --- octue/cloud/emulators/cloud_storage.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/octue/cloud/emulators/cloud_storage.py b/octue/cloud/emulators/cloud_storage.py index 1312a85d6..5aa72a9ce 100644 --- a/octue/cloud/emulators/cloud_storage.py +++ b/octue/cloud/emulators/cloud_storage.py @@ -5,7 +5,7 @@ from contextlib import closing from gcp_storage_emulator.server import create_server -from google.cloud.storage.blob import _API_ACCESS_ENDPOINT +from google.cloud.storage._helpers import _get_default_storage_base_url # Silence the GCP storage emulator logger below `ERROR` level messages. @@ -100,12 +100,18 @@ def stopTestRun(self): del os.environ[self.STORAGE_EMULATOR_HOST_ENVIRONMENT_VARIABLE_NAME] -def mock_generate_signed_url(blob, expiration=datetime.timedelta(days=7), **kwargs): +def mock_generate_signed_url( + blob, + expiration=datetime.timedelta(days=7), + api_access_endpoint=_get_default_storage_base_url(), + **kwargs, +): """Mock generating a signed URL for a Google Cloud Storage blob. Signed URLs can't currently be generated when using workload identity federation, which we use for our CI tests. :param google.cloud.storage.blob.Blob blob: :param datetime.datetime|datetime.timedelta expiration: + :param str api_access_endpoint: :return str: """ mock_signed_query_parameter = ( @@ -113,5 +119,5 @@ def mock_generate_signed_url(blob, expiration=datetime.timedelta(days=7), **kwar f"roject.iam.gserviceaccount.com&Signature=mock-signature" ) - base_url = "/".join((kwargs.get("api_access_endpoint", _API_ACCESS_ENDPOINT), blob.bucket.name, blob.name)) + base_url = "/".join((api_access_endpoint, blob.bucket.name, blob.name)) return base_url + mock_signed_query_parameter From 06123db710ef9e091481f21a8af56a3abf4ad2d4 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 6 Mar 2024 13:40:45 +0000 Subject: [PATCH 019/169] FIX: Add new `Child.ask` arguments to mock and emulator --- octue/cloud/emulators/_pub_sub.py | 2 ++ octue/cloud/emulators/child.py | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/octue/cloud/emulators/_pub_sub.py b/octue/cloud/emulators/_pub_sub.py index f4ace206e..7b43f4235 100644 --- a/octue/cloud/emulators/_pub_sub.py +++ b/octue/cloud/emulators/_pub_sub.py @@ -320,6 +320,7 @@ def ask( save_diagnostics="SAVE_DIAGNOSTICS_ON_CRASH", question_uuid=None, push_endpoint=None, + bigquery_table_id=None, timeout=86400, parent_sdk_version=importlib.metadata.version("octue"), ): @@ -348,6 +349,7 @@ def ask( save_diagnostics=save_diagnostics, question_uuid=question_uuid, push_endpoint=push_endpoint, + bigquery_table_id=bigquery_table_id, timeout=timeout, ) diff --git a/octue/cloud/emulators/child.py b/octue/cloud/emulators/child.py index 71d59a61b..daec2d2b0 100644 --- a/octue/cloud/emulators/child.py +++ b/octue/cloud/emulators/child.py @@ -93,6 +93,8 @@ def ask( handle_monitor_message=None, record_messages=True, question_uuid=None, + push_endpoint=None, + bigquery_table_id=None, timeout=86400, ): """Ask the child emulator a question and receive its emulated response messages. Unlike a real child, the input @@ -106,6 +108,8 @@ def ask( :param callable|None handle_monitor_message: a function to handle monitor messages (e.g. send them to an endpoint for plotting or displaying) - this function should take a single JSON-compatible python primitive as an argument (note that this could be an array or object) :param bool record_messages: if `True`, record messages received from the child in the `received_messages` property :param str|None question_uuid: the UUID to use for the question if a specific one is needed; a UUID is generated if not + :param str|None push_endpoint: if answers to the question should be pushed to an endpoint, provide its URL here (the returned subscription will be a push subscription); if not, leave this as `None` + :param str|None bigquery_table_id: if answers to the questions should be written to BigQuery, provide the ID of the table here (e.g. "your-project.your-dataset.your-table") (the returned subscription will be a BigQuery subscription); if not, leave this as `None` :param float timeout: time in seconds to wait for an answer before raising a timeout error :raise TimeoutError: if the timeout is exceeded while waiting for an answer :return dict: a dictionary containing the keys "output_values" and "output_manifest" @@ -120,6 +124,8 @@ def ask( subscribe_to_logs=subscribe_to_logs, allow_local_files=allow_local_files, question_uuid=question_uuid, + push_endpoint=push_endpoint, + bigquery_table_id=bigquery_table_id, ) return self._parent.wait_for_answer( From 40797032a542d8287c83ed606c142633084800d8 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 6 Mar 2024 13:48:20 +0000 Subject: [PATCH 020/169] ENH: Allow choice of BigQuery table fields returned --- octue/cloud/pub_sub/bigquery.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/octue/cloud/pub_sub/bigquery.py b/octue/cloud/pub_sub/bigquery.py index 8df3678e5..cec069209 100644 --- a/octue/cloud/pub_sub/bigquery.py +++ b/octue/cloud/pub_sub/bigquery.py @@ -1,7 +1,13 @@ from google.cloud import bigquery -def get_events(table_id, question_uuid, limit=1000): +def get_events( + table_id, + question_uuid, + limit=1000, + include_attributes=False, + include_pub_sub_metadata=False, +): """Get Octue service events for a question from a Google BigQuery table. :param str table_id: the full ID of the table e.g. "your-project.your-dataset.your-table" @@ -10,9 +16,16 @@ def get_events(table_id, question_uuid, limit=1000): :return list(dict): the events for the question """ client = bigquery.Client() + fields = ["data"] + + if include_attributes: + fields.append("attributes") + + if include_pub_sub_metadata: + fields.extend(("subscription_name", "message_id", "publish_time")) query = f""" - SELECT * FROM `{table_id}` + SELECT {", ".join(fields)} FROM `{table_id}` WHERE CONTAINS_SUBSTR(subscription_name, @question_uuid) ORDER BY `publish_time` LIMIT @limit From 6c4479ee9a7368eaf2a2407fe699db5970209078 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 6 Mar 2024 14:06:19 +0000 Subject: [PATCH 021/169] ENH: Allow choice of event kinds to return from BigQuery --- octue/cloud/pub_sub/bigquery.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/octue/cloud/pub_sub/bigquery.py b/octue/cloud/pub_sub/bigquery.py index cec069209..9dc63a1cb 100644 --- a/octue/cloud/pub_sub/bigquery.py +++ b/octue/cloud/pub_sub/bigquery.py @@ -1,9 +1,13 @@ from google.cloud import bigquery +VALID_EVENT_KINDS = {"delivery_acknowledgement", "heartbeat", "log_record", "monitor_message", "exception", "result"} + + def get_events( table_id, question_uuid, + kind=None, limit=1000, include_attributes=False, include_pub_sub_metadata=False, @@ -12,9 +16,20 @@ def get_events( :param str table_id: the full ID of the table e.g. "your-project.your-dataset.your-table" :param str question_uuid: the UUID of the question to get the events for - :param int limit: the maximum number of events to return. + :param str|None kind: the kind of event to get; if `None`, all event kinds are returned + :param int limit: the maximum number of events to return + :param bool include_attributes: if `True`, include the event attributes + :param bool include_pub_sub_metadata: if `True`, include Pub/Sub metadata :return list(dict): the events for the question """ + if kind: + if kind not in VALID_EVENT_KINDS: + raise ValueError(f"`kind` must be one of {VALID_EVENT_KINDS!r}; received {kind!r}.") + + kind_condition = f'AND JSON_EXTRACT_SCALAR(data, "$.kind") = "{kind}"' + else: + kind_condition = "" + client = bigquery.Client() fields = ["data"] @@ -27,6 +42,7 @@ def get_events( query = f""" SELECT {", ".join(fields)} FROM `{table_id}` WHERE CONTAINS_SUBSTR(subscription_name, @question_uuid) + {kind_condition} ORDER BY `publish_time` LIMIT @limit """ From 344e9fb1a572321c4a75f1073b4d38ed13adc613 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 6 Mar 2024 14:32:13 +0000 Subject: [PATCH 022/169] REF: Remove excess whitespace from BigQuery queries --- octue/cloud/pub_sub/bigquery.py | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/octue/cloud/pub_sub/bigquery.py b/octue/cloud/pub_sub/bigquery.py index 9dc63a1cb..2e87fa38f 100644 --- a/octue/cloud/pub_sub/bigquery.py +++ b/octue/cloud/pub_sub/bigquery.py @@ -1,4 +1,4 @@ -from google.cloud import bigquery +from google.cloud.bigquery import Client, QueryJobConfig, ScalarQueryParameter VALID_EVENT_KINDS = {"delivery_acknowledgement", "heartbeat", "log_record", "monitor_message", "exception", "result"} @@ -26,11 +26,11 @@ def get_events( if kind not in VALID_EVENT_KINDS: raise ValueError(f"`kind` must be one of {VALID_EVENT_KINDS!r}; received {kind!r}.") - kind_condition = f'AND JSON_EXTRACT_SCALAR(data, "$.kind") = "{kind}"' + event_kind_condition = [f'AND JSON_EXTRACT_SCALAR(data, "$.kind") = "{kind}"'] else: - kind_condition = "" + event_kind_condition = [] - client = bigquery.Client() + client = Client() fields = ["data"] if include_attributes: @@ -39,18 +39,20 @@ def get_events( if include_pub_sub_metadata: fields.extend(("subscription_name", "message_id", "publish_time")) - query = f""" - SELECT {", ".join(fields)} FROM `{table_id}` - WHERE CONTAINS_SUBSTR(subscription_name, @question_uuid) - {kind_condition} - ORDER BY `publish_time` - LIMIT @limit - """ + query = "\n".join( + [ + f"SELECT {', '.join(fields)} FROM `{table_id}`", + "WHERE CONTAINS_SUBSTR(subscription_name, @question_uuid)", + *event_kind_condition, + "ORDER BY publish_time", + "LIMIT @limit", + ] + ) - job_config = bigquery.QueryJobConfig( + job_config = QueryJobConfig( query_parameters=[ - bigquery.ScalarQueryParameter("question_uuid", "STRING", question_uuid), - bigquery.ScalarQueryParameter("limit", "INTEGER", limit), + ScalarQueryParameter("question_uuid", "STRING", question_uuid), + ScalarQueryParameter("limit", "INTEGER", limit), ] ) From e5f184da36601bc13ff942988c56607cd6260827 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 6 Mar 2024 14:33:51 +0000 Subject: [PATCH 023/169] TST: Test BigQuery queries --- tests/cloud/pub_sub/test_bigquery.py | 55 ++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 tests/cloud/pub_sub/test_bigquery.py diff --git a/tests/cloud/pub_sub/test_bigquery.py b/tests/cloud/pub_sub/test_bigquery.py new file mode 100644 index 000000000..301d0f778 --- /dev/null +++ b/tests/cloud/pub_sub/test_bigquery.py @@ -0,0 +1,55 @@ +from unittest import TestCase +from unittest.mock import patch + +from octue.cloud.pub_sub.bigquery import get_events + + +class TestBigQuery(TestCase): + def test_error_raised_if_event_kind_invalid(self): + """Test that an error is raised if the event kind is invalid.""" + with self.assertRaises(ValueError): + get_events(table_id="blah", question_uuid="blah", kind="frisbee_tournament") + + def test_without_kind(self): + """Test the query used to retrieve events of all kinds.""" + with patch("octue.cloud.pub_sub.bigquery.Client") as mock_client: + get_events(table_id="blah", question_uuid="blah") + + self.assertEqual( + mock_client.mock_calls[1].args[0], + "SELECT data FROM `blah`\nWHERE CONTAINS_SUBSTR(subscription_name, @question_uuid)\nORDER BY publish_time\n" + "LIMIT @limit", + ) + + def test_with_kind(self): + """Test the query used to retrieve events of a specific kind.""" + with patch("octue.cloud.pub_sub.bigquery.Client") as mock_client: + get_events(table_id="blah", question_uuid="blah", kind="result") + + self.assertEqual( + mock_client.mock_calls[1].args[0], + "SELECT data FROM `blah`\nWHERE CONTAINS_SUBSTR(subscription_name, @question_uuid)\n" + 'AND JSON_EXTRACT_SCALAR(data, "$.kind") = "result"\nORDER BY publish_time\nLIMIT @limit', + ) + + def test_with_attributes(self): + """Test the query used to retrieve attributes in addition to events.""" + with patch("octue.cloud.pub_sub.bigquery.Client") as mock_client: + get_events(table_id="blah", question_uuid="blah", include_attributes=True) + + self.assertEqual( + mock_client.mock_calls[1].args[0], + "SELECT data, attributes FROM `blah`\nWHERE CONTAINS_SUBSTR(subscription_name, @question_uuid)\n" + "ORDER BY publish_time\nLIMIT @limit", + ) + + def test_with_pub_sub_metadata(self): + """Test the query used to retrieve Pub/Sub metadata in addition to events.""" + with patch("octue.cloud.pub_sub.bigquery.Client") as mock_client: + get_events(table_id="blah", question_uuid="blah", include_pub_sub_metadata=True) + + self.assertEqual( + mock_client.mock_calls[1].args[0], + "SELECT data, subscription_name, message_id, publish_time FROM `blah`\n" + "WHERE CONTAINS_SUBSTR(subscription_name, @question_uuid)\nORDER BY publish_time\nLIMIT @limit", + ) From b71989535eecc579c98f85266fe5848fd0dddafd Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 6 Mar 2024 15:43:32 +0000 Subject: [PATCH 024/169] OPS: Remove default bigquery table expiry --- terraform/bigquery.tf | 1 - 1 file changed, 1 deletion(-) diff --git a/terraform/bigquery.tf b/terraform/bigquery.tf index cd76b2cac..448594fce 100644 --- a/terraform/bigquery.tf +++ b/terraform/bigquery.tf @@ -2,7 +2,6 @@ resource "google_bigquery_dataset" "test_dataset" { dataset_id = "octue_sdk_python_test_dataset" description = "A dataset for testing BigQuery subscriptions for the Octue SDK." location = "EU" - default_table_expiration_ms = 3600000 labels = { env = "default" From f4e8ceb3e6a58f965dbc2bb31dca14dfe87fd4ed Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 6 Mar 2024 16:06:46 +0000 Subject: [PATCH 025/169] ENH: Sort events from BigQuery by message number --- octue/cloud/pub_sub/bigquery.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/octue/cloud/pub_sub/bigquery.py b/octue/cloud/pub_sub/bigquery.py index 2e87fa38f..1b0f0a98c 100644 --- a/octue/cloud/pub_sub/bigquery.py +++ b/octue/cloud/pub_sub/bigquery.py @@ -1,3 +1,5 @@ +import ast + from google.cloud.bigquery import Client, QueryJobConfig, ScalarQueryParameter @@ -60,4 +62,9 @@ def get_events( rows = query_job.result() messages = rows.to_dataframe() + # Order messages by the message number. + if isinstance(messages.at[0, "attributes"], str): + messages["attributes"] = messages["attributes"].map(ast.literal_eval) + + messages = messages.iloc[messages["attributes"].str.get("message_number").astype(str).argsort()] return messages.to_dict(orient="records") From 488957ed2e9a01c9c5567b561cb7541d940651cd Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 6 Mar 2024 16:08:18 +0000 Subject: [PATCH 026/169] FIX: Always include attributes with events from bigquery --- octue/cloud/pub_sub/bigquery.py | 15 ++------------- tests/cloud/pub_sub/test_bigquery.py | 19 ++++--------------- 2 files changed, 6 insertions(+), 28 deletions(-) diff --git a/octue/cloud/pub_sub/bigquery.py b/octue/cloud/pub_sub/bigquery.py index 1b0f0a98c..97d501f61 100644 --- a/octue/cloud/pub_sub/bigquery.py +++ b/octue/cloud/pub_sub/bigquery.py @@ -6,21 +6,13 @@ VALID_EVENT_KINDS = {"delivery_acknowledgement", "heartbeat", "log_record", "monitor_message", "exception", "result"} -def get_events( - table_id, - question_uuid, - kind=None, - limit=1000, - include_attributes=False, - include_pub_sub_metadata=False, -): +def get_events(table_id, question_uuid, kind=None, limit=1000, include_pub_sub_metadata=False): """Get Octue service events for a question from a Google BigQuery table. :param str table_id: the full ID of the table e.g. "your-project.your-dataset.your-table" :param str question_uuid: the UUID of the question to get the events for :param str|None kind: the kind of event to get; if `None`, all event kinds are returned :param int limit: the maximum number of events to return - :param bool include_attributes: if `True`, include the event attributes :param bool include_pub_sub_metadata: if `True`, include Pub/Sub metadata :return list(dict): the events for the question """ @@ -33,10 +25,7 @@ def get_events( event_kind_condition = [] client = Client() - fields = ["data"] - - if include_attributes: - fields.append("attributes") + fields = ["data", "attributes"] if include_pub_sub_metadata: fields.extend(("subscription_name", "message_id", "publish_time")) diff --git a/tests/cloud/pub_sub/test_bigquery.py b/tests/cloud/pub_sub/test_bigquery.py index 301d0f778..9e9a21eb8 100644 --- a/tests/cloud/pub_sub/test_bigquery.py +++ b/tests/cloud/pub_sub/test_bigquery.py @@ -17,8 +17,8 @@ def test_without_kind(self): self.assertEqual( mock_client.mock_calls[1].args[0], - "SELECT data FROM `blah`\nWHERE CONTAINS_SUBSTR(subscription_name, @question_uuid)\nORDER BY publish_time\n" - "LIMIT @limit", + "SELECT data, attributes FROM `blah`\nWHERE CONTAINS_SUBSTR(subscription_name, @question_uuid)\n" + "ORDER BY publish_time\nLIMIT @limit", ) def test_with_kind(self): @@ -26,21 +26,10 @@ def test_with_kind(self): with patch("octue.cloud.pub_sub.bigquery.Client") as mock_client: get_events(table_id="blah", question_uuid="blah", kind="result") - self.assertEqual( - mock_client.mock_calls[1].args[0], - "SELECT data FROM `blah`\nWHERE CONTAINS_SUBSTR(subscription_name, @question_uuid)\n" - 'AND JSON_EXTRACT_SCALAR(data, "$.kind") = "result"\nORDER BY publish_time\nLIMIT @limit', - ) - - def test_with_attributes(self): - """Test the query used to retrieve attributes in addition to events.""" - with patch("octue.cloud.pub_sub.bigquery.Client") as mock_client: - get_events(table_id="blah", question_uuid="blah", include_attributes=True) - self.assertEqual( mock_client.mock_calls[1].args[0], "SELECT data, attributes FROM `blah`\nWHERE CONTAINS_SUBSTR(subscription_name, @question_uuid)\n" - "ORDER BY publish_time\nLIMIT @limit", + 'AND JSON_EXTRACT_SCALAR(data, "$.kind") = "result"\nORDER BY publish_time\nLIMIT @limit', ) def test_with_pub_sub_metadata(self): @@ -50,6 +39,6 @@ def test_with_pub_sub_metadata(self): self.assertEqual( mock_client.mock_calls[1].args[0], - "SELECT data, subscription_name, message_id, publish_time FROM `blah`\n" + "SELECT data, attributes, subscription_name, message_id, publish_time FROM `blah`\n" "WHERE CONTAINS_SUBSTR(subscription_name, @question_uuid)\nORDER BY publish_time\nLIMIT @limit", ) From 5187bf7115504ae00dc624e2fd9f243df2c8bf3a Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 6 Mar 2024 16:31:49 +0000 Subject: [PATCH 027/169] REF: Factor out event handlers into `EventHandler` class skipci --- octue/cloud/events/__init__.py | 0 octue/cloud/events/event_handler.py | 163 +++++++++++++++++++++++++ octue/cloud/pub_sub/message_handler.py | 152 ++--------------------- 3 files changed, 174 insertions(+), 141 deletions(-) create mode 100644 octue/cloud/events/__init__.py create mode 100644 octue/cloud/events/event_handler.py diff --git a/octue/cloud/events/__init__.py b/octue/cloud/events/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/octue/cloud/events/event_handler.py b/octue/cloud/events/event_handler.py new file mode 100644 index 000000000..c8540cea0 --- /dev/null +++ b/octue/cloud/events/event_handler.py @@ -0,0 +1,163 @@ +import logging +import os +import re +from datetime import datetime + +from octue.cloud import EXCEPTIONS_MAPPING +from octue.cloud.validation import SERVICE_COMMUNICATION_SCHEMA +from octue.definitions import GOOGLE_COMPUTE_PROVIDERS +from octue.log_handlers import COLOUR_PALETTE +from octue.resources.manifest import Manifest + + +logger = logging.getLogger(__name__) + + +if os.environ.get("COMPUTE_PROVIDER", "UNKNOWN") in GOOGLE_COMPUTE_PROVIDERS: + # Google Cloud logs don't support colour currently - provide a no-operation function. + colourise = lambda string, text_colour=None, background_colour=None: string +else: + from octue.utils.colour import colourise + + +class EventHandler: + question_uuid: str + + def __init__( + self, + receiving_service, + handle_monitor_message=None, + record_messages=True, + service_name="REMOTE", + message_handlers=None, + schema=SERVICE_COMMUNICATION_SCHEMA, + ): + self.receiving_service = receiving_service + self.handle_monitor_message = handle_monitor_message + self.record_messages = record_messages + self.service_name = service_name + self.schema = schema + + self.handled_messages = [] + self._previous_message_number = -1 + + self._message_handlers = message_handlers or { + "delivery_acknowledgement": self._handle_delivery_acknowledgement, + "heartbeat": self._handle_heartbeat, + "monitor_message": self._handle_monitor_message, + "log_record": self._handle_log_message, + "exception": self._handle_exception, + "result": self._handle_result, + } + + self._log_message_colours = [COLOUR_PALETTE[1], *COLOUR_PALETTE[3:]] + + def _handle_message(self, message): + """Pass a message to its handler and update the previous message number. + + :param dict message: + :return dict|None: + """ + self._previous_message_number += 1 + + if self.record_messages: + self.handled_messages.append(message) + + handler = self._message_handlers[message["kind"]] + return handler(message) + + def _handle_delivery_acknowledgement(self, message): + """Mark the question as delivered to prevent resending it. + + :param dict message: + :return None: + """ + logger.info("%r's question was delivered at %s.", self.receiving_service, message["datetime"]) + + def _handle_heartbeat(self, message): + """Record the time the heartbeat was received. + + :param dict message: + :return None: + """ + self._last_heartbeat = datetime.now() + logger.info("Heartbeat received from service %r for question %r.", self.service_name, self.question_uuid) + + def _handle_monitor_message(self, message): + """Send a monitor message to the handler if one has been provided. + + :param dict message: + :return None: + """ + logger.debug("%r received a monitor message.", self.receiving_service) + + if self.handle_monitor_message is not None: + self.handle_monitor_message(message["data"]) + + def _handle_log_message(self, message): + """Deserialise the message into a log record and pass it to the local log handlers, adding [] to + the start of the log message. + + :param dict message: + :return None: + """ + record = logging.makeLogRecord(message["log_record"]) + + # Add information about the immediate child sending the message and colour it with the first colour in the + # colour palette. + immediate_child_analysis_section = colourise( + f"[{self.service_name} | analysis-{self.question_uuid}]", + text_colour=self._log_message_colours[0], + ) + + # Colour any analysis sections from children of the immediate child with the rest of the colour palette. + subchild_analysis_sections = [section.strip("[") for section in re.split("] ", record.msg)] + final_message = subchild_analysis_sections.pop(-1) + + for i in range(len(subchild_analysis_sections)): + subchild_analysis_sections[i] = colourise( + "[" + subchild_analysis_sections[i] + "]", + text_colour=self._log_message_colours[1:][i % len(self._log_message_colours[1:])], + ) + + record.msg = " ".join([immediate_child_analysis_section, *subchild_analysis_sections, final_message]) + logger.handle(record) + + def _handle_exception(self, message): + """Raise the exception from the responding service that is serialised in `data`. + + :param dict message: + :raise Exception: + :return None: + """ + exception_message = "\n\n".join( + ( + message["exception_message"], + f"The following traceback was captured from the remote service {self.service_name!r}:", + "".join(message["exception_traceback"]), + ) + ) + + try: + exception_type = EXCEPTIONS_MAPPING[message["exception_type"]] + + # Allow unknown exception types to still be raised. + except KeyError: + exception_type = type(message["exception_type"], (Exception,), {}) + + raise exception_type(exception_message) + + def _handle_result(self, message): + """Convert the result to the correct form, deserialising the output manifest if it is present in the message. + + :param dict message: + :return dict: + """ + logger.info("%r received an answer to question %r.", self.receiving_service, self.question_uuid) + + if message.get("output_manifest"): + output_manifest = Manifest.deserialise(message["output_manifest"]) + else: + output_manifest = None + + return {"output_values": message.get("output_values"), "output_manifest": output_manifest} diff --git a/octue/cloud/pub_sub/message_handler.py b/octue/cloud/pub_sub/message_handler.py index 296d35e47..35bad4d56 100644 --- a/octue/cloud/pub_sub/message_handler.py +++ b/octue/cloud/pub_sub/message_handler.py @@ -1,29 +1,18 @@ import importlib.metadata import logging import math -import os -import re import time from datetime import datetime, timedelta from google.api_core import retry from google.cloud.pubsub_v1 import SubscriberClient -from octue.cloud import EXCEPTIONS_MAPPING +from octue.cloud.events.event_handler import EventHandler from octue.cloud.pub_sub.events import extract_event_and_attributes_from_pub_sub from octue.cloud.validation import SERVICE_COMMUNICATION_SCHEMA, is_event_valid -from octue.definitions import GOOGLE_COMPUTE_PROVIDERS -from octue.log_handlers import COLOUR_PALETTE -from octue.resources.manifest import Manifest from octue.utils.threads import RepeatingTimer -if os.environ.get("COMPUTE_PROVIDER", "UNKNOWN") in GOOGLE_COMPUTE_PROVIDERS: - # Google Cloud logs don't support colour currently - provide a no-operation function. - colourise = lambda string, text_colour=None, background_colour=None: string -else: - from octue.utils.colour import colourise - logger = logging.getLogger(__name__) @@ -31,7 +20,7 @@ PARENT_SDK_VERSION = importlib.metadata.version("octue") -class OrderedMessageHandler: +class OrderedMessageHandler(EventHandler): """A handler for Google Pub/Sub messages received via a pull subscription that ensures messages are handled in the order they were sent. @@ -58,17 +47,20 @@ def __init__( skip_missing_messages_after=10, ): self.subscription = subscription - self.receiving_service = receiving_service - self.handle_monitor_message = handle_monitor_message - self.record_messages = record_messages - self.service_name = service_name - self.schema = schema + + super().__init__( + receiving_service, + handle_monitor_message=handle_monitor_message, + record_messages=record_messages, + service_name=service_name, + message_handlers=message_handlers, + schema=schema, + ) self.skip_missing_messages_after = skip_missing_messages_after self._missing_message_detection_time = None self.question_uuid = self.subscription.path.split(".")[-1] - self.handled_messages = [] self.waiting_messages = None self._subscriber = SubscriberClient() self._child_sdk_version = None @@ -76,20 +68,8 @@ def __init__( self._last_heartbeat = None self._alive = True self._start_time = None - self._previous_message_number = -1 self._earliest_waiting_message_number = math.inf - self._message_handlers = message_handlers or { - "delivery_acknowledgement": self._handle_delivery_acknowledgement, - "heartbeat": self._handle_heartbeat, - "monitor_message": self._handle_monitor_message, - "log_record": self._handle_log_message, - "exception": self._handle_exception, - "result": self._handle_result, - } - - self._log_message_colours = [COLOUR_PALETTE[1], *COLOUR_PALETTE[3:]] - @property def total_run_time(self): """Get the amount of time elapsed since `self.handle_messages` was called. If it hasn't been called yet, it will @@ -346,113 +326,3 @@ def _skip_to_earliest_waiting_message(self): ) return message - - def _handle_message(self, message): - """Pass a message to its handler and update the previous message number. - - :param dict message: - :return dict|None: - """ - self._previous_message_number += 1 - - if self.record_messages: - self.handled_messages.append(message) - - handler = self._message_handlers[message["kind"]] - return handler(message) - - def _handle_delivery_acknowledgement(self, message): - """Mark the question as delivered to prevent resending it. - - :param dict message: - :return None: - """ - logger.info("%r's question was delivered at %s.", self.receiving_service, message["datetime"]) - - def _handle_heartbeat(self, message): - """Record the time the heartbeat was received. - - :param dict message: - :return None: - """ - self._last_heartbeat = datetime.now() - logger.info("Heartbeat received from service %r for question %r.", self.service_name, self.question_uuid) - - def _handle_monitor_message(self, message): - """Send a monitor message to the handler if one has been provided. - - :param dict message: - :return None: - """ - logger.debug("%r received a monitor message.", self.receiving_service) - - if self.handle_monitor_message is not None: - self.handle_monitor_message(message["data"]) - - def _handle_log_message(self, message): - """Deserialise the message into a log record and pass it to the local log handlers, adding [] to - the start of the log message. - - :param dict message: - :return None: - """ - record = logging.makeLogRecord(message["log_record"]) - - # Add information about the immediate child sending the message and colour it with the first colour in the - # colour palette. - immediate_child_analysis_section = colourise( - f"[{self.service_name} | analysis-{self.question_uuid}]", - text_colour=self._log_message_colours[0], - ) - - # Colour any analysis sections from children of the immediate child with the rest of the colour palette. - subchild_analysis_sections = [section.strip("[") for section in re.split("] ", record.msg)] - final_message = subchild_analysis_sections.pop(-1) - - for i in range(len(subchild_analysis_sections)): - subchild_analysis_sections[i] = colourise( - "[" + subchild_analysis_sections[i] + "]", - text_colour=self._log_message_colours[1:][i % len(self._log_message_colours[1:])], - ) - - record.msg = " ".join([immediate_child_analysis_section, *subchild_analysis_sections, final_message]) - logger.handle(record) - - def _handle_exception(self, message): - """Raise the exception from the responding service that is serialised in `data`. - - :param dict message: - :raise Exception: - :return None: - """ - exception_message = "\n\n".join( - ( - message["exception_message"], - f"The following traceback was captured from the remote service {self.service_name!r}:", - "".join(message["exception_traceback"]), - ) - ) - - try: - exception_type = EXCEPTIONS_MAPPING[message["exception_type"]] - - # Allow unknown exception types to still be raised. - except KeyError: - exception_type = type(message["exception_type"], (Exception,), {}) - - raise exception_type(exception_message) - - def _handle_result(self, message): - """Convert the result to the correct form, deserialising the output manifest if it is present in the message. - - :param dict message: - :return dict: - """ - logger.info("%r received an answer to question %r.", self.receiving_service, self.question_uuid) - - if message.get("output_manifest"): - output_manifest = Manifest.deserialise(message["output_manifest"]) - else: - output_manifest = None - - return {"output_values": message.get("output_values"), "output_manifest": output_manifest} From 9987f81447fed0528ae92aae839ed671315fd3a1 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 6 Mar 2024 16:33:36 +0000 Subject: [PATCH 028/169] REF: Move `validation` module into `octue.cloud.events` subpackage --- octue/cloud/events/event_handler.py | 2 +- octue/cloud/{ => events}/validation.py | 0 octue/cloud/pub_sub/message_handler.py | 2 +- octue/cloud/pub_sub/service.py | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename octue/cloud/{ => events}/validation.py (100%) diff --git a/octue/cloud/events/event_handler.py b/octue/cloud/events/event_handler.py index c8540cea0..2703d84bb 100644 --- a/octue/cloud/events/event_handler.py +++ b/octue/cloud/events/event_handler.py @@ -4,7 +4,7 @@ from datetime import datetime from octue.cloud import EXCEPTIONS_MAPPING -from octue.cloud.validation import SERVICE_COMMUNICATION_SCHEMA +from octue.cloud.events.validation import SERVICE_COMMUNICATION_SCHEMA from octue.definitions import GOOGLE_COMPUTE_PROVIDERS from octue.log_handlers import COLOUR_PALETTE from octue.resources.manifest import Manifest diff --git a/octue/cloud/validation.py b/octue/cloud/events/validation.py similarity index 100% rename from octue/cloud/validation.py rename to octue/cloud/events/validation.py diff --git a/octue/cloud/pub_sub/message_handler.py b/octue/cloud/pub_sub/message_handler.py index 35bad4d56..d7ad11c01 100644 --- a/octue/cloud/pub_sub/message_handler.py +++ b/octue/cloud/pub_sub/message_handler.py @@ -8,8 +8,8 @@ from google.cloud.pubsub_v1 import SubscriberClient from octue.cloud.events.event_handler import EventHandler +from octue.cloud.events.validation import SERVICE_COMMUNICATION_SCHEMA, is_event_valid from octue.cloud.pub_sub.events import extract_event_and_attributes_from_pub_sub -from octue.cloud.validation import SERVICE_COMMUNICATION_SCHEMA, is_event_valid from octue.utils.threads import RepeatingTimer diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index 6eba37151..d49e5b2e8 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -14,6 +14,7 @@ from google.cloud import pubsub_v1 import octue.exceptions +from octue.cloud.events.validation import raise_if_event_is_invalid from octue.cloud.pub_sub import Subscription, Topic from octue.cloud.pub_sub.events import extract_event_and_attributes_from_pub_sub from octue.cloud.pub_sub.logging import GooglePubSubHandler @@ -27,7 +28,6 @@ split_service_id, validate_sruid, ) -from octue.cloud.validation import raise_if_event_is_invalid from octue.compatibility import warn_if_incompatible from octue.utils.dictionaries import make_minimal_dictionary from octue.utils.encoders import OctueJSONEncoder From 1bc94074c6ce8aae394eda53834d9ec4900b5a34 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 6 Mar 2024 16:56:26 +0000 Subject: [PATCH 029/169] REF: Factor out more from message handler into event handler --- octue/cloud/events/event_handler.py | 130 ++++++++++++++++++++++++- octue/cloud/pub_sub/message_handler.py | 120 +---------------------- 2 files changed, 133 insertions(+), 117 deletions(-) diff --git a/octue/cloud/events/event_handler.py b/octue/cloud/events/event_handler.py index 2703d84bb..f5674c5fa 100644 --- a/octue/cloud/events/event_handler.py +++ b/octue/cloud/events/event_handler.py @@ -1,10 +1,14 @@ +import abc +import importlib.metadata import logging +import math import os import re +import time from datetime import datetime from octue.cloud import EXCEPTIONS_MAPPING -from octue.cloud.events.validation import SERVICE_COMMUNICATION_SCHEMA +from octue.cloud.events.validation import SERVICE_COMMUNICATION_SCHEMA, is_event_valid from octue.definitions import GOOGLE_COMPUTE_PROVIDERS from octue.log_handlers import COLOUR_PALETTE from octue.resources.manifest import Manifest @@ -20,6 +24,9 @@ from octue.utils.colour import colourise +PARENT_SDK_VERSION = importlib.metadata.version("octue") + + class EventHandler: question_uuid: str @@ -31,6 +38,7 @@ def __init__( service_name="REMOTE", message_handlers=None, schema=SERVICE_COMMUNICATION_SCHEMA, + skip_missing_messages_after=10, ): self.receiving_service = receiving_service self.handle_monitor_message = handle_monitor_message @@ -38,8 +46,14 @@ def __init__( self.service_name = service_name self.schema = schema + self.waiting_messages = None self.handled_messages = [] self._previous_message_number = -1 + self._child_sdk_version = None + + self.skip_missing_messages_after = skip_missing_messages_after + self._missing_message_detection_time = None + self._earliest_waiting_message_number = math.inf self._message_handlers = message_handlers or { "delivery_acknowledgement": self._handle_delivery_acknowledgement, @@ -52,6 +66,120 @@ def __init__( self._log_message_colours = [COLOUR_PALETTE[1], *COLOUR_PALETTE[3:]] + @property + def time_since_missing_message(self): + """Get the amount of time elapsed since the last missing message was detected. If no missing messages have been + detected or they've already been skipped past, `None` is returned. + + :return float|None: + """ + if self._missing_message_detection_time is None: + return None + + return time.perf_counter() - self._missing_message_detection_time + + @abc.abstractmethod + def _extract_event_and_attributes(self, event): + pass + + def _extract_and_enqueue_event(self, event): + """Extract an event from the Pub/Sub message and add it to `self.waiting_messages`. + + :param dict event: + :return None: + """ + logger.debug("%r received a message related to question %r.", self.receiving_service, self.question_uuid) + event, attributes = self._extract_event_and_attributes(event) + + if not is_event_valid( + event=event, + attributes=attributes, + receiving_service=self.receiving_service, + parent_sdk_version=PARENT_SDK_VERSION, + child_sdk_version=attributes.get("version"), + schema=self.schema, + ): + return + + # Get the child's Octue SDK version from the first message. + if not self._child_sdk_version: + self._child_sdk_version = attributes["version"] + + message_number = attributes["message_number"] + + if message_number in self.waiting_messages: + logger.warning( + "%r: Message with duplicate message number %d received for question %s - overwriting original message.", + self.receiving_service, + message_number, + self.question_uuid, + ) + + self.waiting_messages[message_number] = event + + def _attempt_to_handle_waiting_messages(self): + """Attempt to handle messages waiting in `self.waiting_messages`. If these messages aren't consecutive to the + last handled message (i.e. if messages have been received out of order and the next in-order message hasn't been + received yet), just return. After the missing message wait time has passed, if this set of missing messages + haven't arrived but subsequent ones have, skip to the earliest waiting message and continue from there. + + :return any|None: either a non-`None` result from a message handler or `None` if nothing was returned by the message handlers or if the next in-order message hasn't been received yet + """ + while self.waiting_messages: + try: + # If the next consecutive message has been received: + message = self.waiting_messages.pop(self._previous_message_number + 1) + + # If the next consecutive message hasn't been received: + except KeyError: + # Start the missing message timer if it isn't already running. + if self._missing_message_detection_time is None: + self._missing_message_detection_time = time.perf_counter() + + if self.time_since_missing_message > self.skip_missing_messages_after: + message = self._skip_to_earliest_waiting_message() + + # Declare there are no more missing messages. + self._missing_message_detection_time = None + + if not message: + return + + else: + return + + result = self._handle_message(message) + + if result is not None: + return result + + def _skip_to_earliest_waiting_message(self): + """Get the earliest waiting message and set the message handler up to continue from it. + + :return dict|None: + """ + try: + message = self.waiting_messages.pop(self._earliest_waiting_message_number) + except KeyError: + return + + number_of_missing_messages = self._earliest_waiting_message_number - self._previous_message_number - 1 + + # Let the message handler know it can handle the next earliest message. + self._previous_message_number = self._earliest_waiting_message_number - 1 + + logger.warning( + "%r: %d consecutive messages missing for question %r after %ds - skipping to next earliest waiting message " + "(message %d).", + self.receiving_service, + number_of_missing_messages, + self.question_uuid, + self.skip_missing_messages_after, + self._earliest_waiting_message_number, + ) + + return message + def _handle_message(self, message): """Pass a message to its handler and update the previous message number. diff --git a/octue/cloud/pub_sub/message_handler.py b/octue/cloud/pub_sub/message_handler.py index d7ad11c01..c56a44702 100644 --- a/octue/cloud/pub_sub/message_handler.py +++ b/octue/cloud/pub_sub/message_handler.py @@ -1,6 +1,5 @@ import importlib.metadata import logging -import math import time from datetime import datetime, timedelta @@ -8,7 +7,7 @@ from google.cloud.pubsub_v1 import SubscriberClient from octue.cloud.events.event_handler import EventHandler -from octue.cloud.events.validation import SERVICE_COMMUNICATION_SCHEMA, is_event_valid +from octue.cloud.events.validation import SERVICE_COMMUNICATION_SCHEMA from octue.cloud.pub_sub.events import extract_event_and_attributes_from_pub_sub from octue.utils.threads import RepeatingTimer @@ -55,20 +54,16 @@ def __init__( service_name=service_name, message_handlers=message_handlers, schema=schema, + skip_missing_messages_after=skip_missing_messages_after, ) - self.skip_missing_messages_after = skip_missing_messages_after - self._missing_message_detection_time = None - self.question_uuid = self.subscription.path.split(".")[-1] self.waiting_messages = None self._subscriber = SubscriberClient() - self._child_sdk_version = None self._heartbeat_checker = None self._last_heartbeat = None self._alive = True self._start_time = None - self._earliest_waiting_message_number = math.inf @property def total_run_time(self): @@ -82,18 +77,6 @@ def total_run_time(self): return time.perf_counter() - self._start_time - @property - def time_since_missing_message(self): - """Get the amount of time elapsed since the last missing message was detected. If no missing messages have been - detected or they've already been skipped past, `None` is returned. - - :return float|None: - """ - if self._missing_message_detection_time is None: - return None - - return time.perf_counter() - self._missing_message_detection_time - @property def _time_since_last_heartbeat(self): """Get the time period since the last heartbeat was received. @@ -229,100 +212,5 @@ def _pull_and_enqueue_available_messages(self, timeout): self._earliest_waiting_message_number = min(self.waiting_messages.keys()) - def _extract_and_enqueue_event(self, message): - """Extract an event from the Pub/Sub message and add it to `self.waiting_messages`. - - :param dict message: - :return None: - """ - logger.debug("%r received a message related to question %r.", self.receiving_service, self.question_uuid) - event, attributes = extract_event_and_attributes_from_pub_sub(message.message) - - if not is_event_valid( - event=event, - attributes=attributes, - receiving_service=self.receiving_service, - parent_sdk_version=PARENT_SDK_VERSION, - child_sdk_version=attributes.get("version"), - schema=self.schema, - ): - return - - # Get the child's Octue SDK version from the first message. - if not self._child_sdk_version: - self._child_sdk_version = attributes["version"] - - message_number = attributes["message_number"] - - if message_number in self.waiting_messages: - logger.warning( - "%r: Message with duplicate message number %d received for question %s - overwriting original message.", - self.receiving_service, - message_number, - self.question_uuid, - ) - - self.waiting_messages[message_number] = event - - def _attempt_to_handle_waiting_messages(self): - """Attempt to handle messages waiting in `self.waiting_messages`. If these messages aren't consecutive to the - last handled message (i.e. if messages have been received out of order and the next in-order message hasn't been - received yet), just return. After the missing message wait time has passed, if this set of missing messages - haven't arrived but subsequent ones have, skip to the earliest waiting message and continue from there. - - :return any|None: either a non-`None` result from a message handler or `None` if nothing was returned by the message handlers or if the next in-order message hasn't been received yet - """ - while self.waiting_messages: - try: - # If the next consecutive message has been received: - message = self.waiting_messages.pop(self._previous_message_number + 1) - - # If the next consecutive message hasn't been received: - except KeyError: - # Start the missing message timer if it isn't already running. - if self._missing_message_detection_time is None: - self._missing_message_detection_time = time.perf_counter() - - if self.time_since_missing_message > self.skip_missing_messages_after: - message = self._skip_to_earliest_waiting_message() - - # Declare there are no more missing messages. - self._missing_message_detection_time = None - - if not message: - return - - else: - return - - result = self._handle_message(message) - - if result is not None: - return result - - def _skip_to_earliest_waiting_message(self): - """Get the earliest waiting message and set the message handler up to continue from it. - - :return dict|None: - """ - try: - message = self.waiting_messages.pop(self._earliest_waiting_message_number) - except KeyError: - return - - number_of_missing_messages = self._earliest_waiting_message_number - self._previous_message_number - 1 - - # Let the message handler know it can handle the next earliest message. - self._previous_message_number = self._earliest_waiting_message_number - 1 - - logger.warning( - "%r: %d consecutive messages missing for question %r after %ds - skipping to next earliest waiting message " - "(message %d).", - self.receiving_service, - number_of_missing_messages, - self.question_uuid, - self.skip_missing_messages_after, - self._earliest_waiting_message_number, - ) - - return message + def _extract_event_and_attributes(self, message): + return extract_event_and_attributes_from_pub_sub(message.message) From 4721bec5fcc19c7c51daf745a14af32f06812ece Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 6 Mar 2024 17:04:39 +0000 Subject: [PATCH 030/169] ENH: Rename "data" column to "event" when getting from bigquery --- octue/cloud/pub_sub/bigquery.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/octue/cloud/pub_sub/bigquery.py b/octue/cloud/pub_sub/bigquery.py index 97d501f61..f3e4495d8 100644 --- a/octue/cloud/pub_sub/bigquery.py +++ b/octue/cloud/pub_sub/bigquery.py @@ -56,4 +56,6 @@ def get_events(table_id, question_uuid, kind=None, limit=1000, include_pub_sub_m messages["attributes"] = messages["attributes"].map(ast.literal_eval) messages = messages.iloc[messages["attributes"].str.get("message_number").astype(str).argsort()] + + messages.rename(columns={"data": "event"}, inplace=True) return messages.to_dict(orient="records") From a98196d9ffc4a4a89397738660a037ae4f2fa6ff Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 6 Mar 2024 17:14:29 +0000 Subject: [PATCH 031/169] FIX: Convert JSON to python primitives when getting from bigquery --- octue/cloud/pub_sub/bigquery.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/octue/cloud/pub_sub/bigquery.py b/octue/cloud/pub_sub/bigquery.py index f3e4495d8..242037e5f 100644 --- a/octue/cloud/pub_sub/bigquery.py +++ b/octue/cloud/pub_sub/bigquery.py @@ -1,4 +1,4 @@ -import ast +import json from google.cloud.bigquery import Client, QueryJobConfig, ScalarQueryParameter @@ -51,11 +51,14 @@ def get_events(table_id, question_uuid, kind=None, limit=1000, include_pub_sub_m rows = query_job.result() messages = rows.to_dataframe() - # Order messages by the message number. + # Convert JSON to python primitives. + if isinstance(messages.at[0, "data"], str): + messages["data"] = messages["data"].map(json.loads) + if isinstance(messages.at[0, "attributes"], str): - messages["attributes"] = messages["attributes"].map(ast.literal_eval) + messages["attributes"] = messages["attributes"].map(json.loads) + # Order messages by the message number. messages = messages.iloc[messages["attributes"].str.get("message_number").astype(str).argsort()] - messages.rename(columns={"data": "event"}, inplace=True) return messages.to_dict(orient="records") From 6e6916a58df88dd9120bb97818ad679ab8a8ad43 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 6 Mar 2024 17:35:42 +0000 Subject: [PATCH 032/169] FEA: Add event replayer --- octue/cloud/events/replayer.py | 45 ++++++ tests/cloud/events/__init__.py | 0 tests/cloud/events/test_replayer.py | 37 +++++ tests/data/events.json | 219 ++++++++++++++++++++++++++++ 4 files changed, 301 insertions(+) create mode 100644 octue/cloud/events/replayer.py create mode 100644 tests/cloud/events/__init__.py create mode 100644 tests/cloud/events/test_replayer.py create mode 100644 tests/data/events.json diff --git a/octue/cloud/events/replayer.py b/octue/cloud/events/replayer.py new file mode 100644 index 000000000..f405f8c29 --- /dev/null +++ b/octue/cloud/events/replayer.py @@ -0,0 +1,45 @@ +import logging + +from octue.cloud.events.event_handler import EventHandler +from octue.cloud.events.validation import SERVICE_COMMUNICATION_SCHEMA +from octue.cloud.pub_sub.service import Service +from octue.resources.service_backends import ServiceBackend + + +logger = logging.getLogger(__name__) + + +class EventReplayer(EventHandler): + def __init__( + self, + receiving_service=None, + handle_monitor_message=None, + record_messages=True, + service_name="REMOTE", + message_handlers=None, + schema=SERVICE_COMMUNICATION_SCHEMA, + ): + super().__init__( + receiving_service or Service(backend=ServiceBackend(), service_id="local/local:local"), + handle_monitor_message=handle_monitor_message, + record_messages=record_messages, + service_name=service_name, + message_handlers=message_handlers, + schema=schema, + skip_missing_messages_after=0, + ) + + def handle_events(self, events): + self.question_uuid = events[0]["attributes"]["question_uuid"] + self.waiting_messages = {} + self._previous_message_number = -1 + + for event in events: + self._extract_and_enqueue_event(event) + + self._earliest_waiting_message_number = min(self.waiting_messages.keys()) + return self._attempt_to_handle_waiting_messages() + + def _extract_event_and_attributes(self, event): + event["attributes"]["message_number"] = int(event["attributes"]["message_number"]) + return event["event"], event["attributes"] diff --git a/tests/cloud/events/__init__.py b/tests/cloud/events/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/cloud/events/test_replayer.py b/tests/cloud/events/test_replayer.py new file mode 100644 index 000000000..7bf95ba65 --- /dev/null +++ b/tests/cloud/events/test_replayer.py @@ -0,0 +1,37 @@ +import json +import os +import unittest + +from octue.cloud.events.replayer import EventReplayer +from tests import TEST_BUCKET_NAME, TESTS_DIR + + +class TestEventReplayer(unittest.TestCase): + def test_replay_events(self): + """Test that stored events can be replayed and the result extracted.""" + with open(os.path.join(TESTS_DIR, "data", "events.json")) as f: + events = json.load(f) + + result = EventReplayer().handle_events(events) + self.assertEqual(result["output_values"], [1, 2, 3, 4, 5]) + + output_manifest = result["output_manifest"].to_primitive() + del output_manifest["datasets"]["example_dataset"]["id"] + + self.assertEqual( + output_manifest, + { + "id": "a13713ae-f207-41c6-9e29-0a848ced6039", + "name": None, + "datasets": { + "example_dataset": { + "name": "divergent-strange-gharial-of-pizza", + "tags": {}, + "labels": [], + "path": "https://storage.googleapis.com/octue-sdk-python-test-bucket/example_output_datasets" + "/example_dataset/.signed_metadata_files/divergent-strange-gharial-of-pizza", + "files": [f"gs://{TEST_BUCKET_NAME}/example_output_datasets/example_dataset/output.dat"], + } + }, + }, + ) diff --git a/tests/data/events.json b/tests/data/events.json new file mode 100644 index 000000000..81396a7ac --- /dev/null +++ b/tests/data/events.json @@ -0,0 +1,219 @@ +[ + { + "event": { + "datetime": "2024-03-06T15:44:18.156044", + "kind": "delivery_acknowledgement" + }, + "attributes": { + "message_number": "0", + "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", + "sender_type": "CHILD", + "version": "0.51.0" + } + }, + { + "event": { + "kind": "log_record", + "log_record": { + "args": null, + "created": 1709739858.1877987, + "exc_info": null, + "exc_text": null, + "filename": "app.py", + "funcName": "run", + "levelname": "INFO", + "levelno": 20, + "lineno": 15, + "module": "app", + "msecs": 187.79873847961426, + "msg": "Started example analysis.", + "name": "app", + "pathname": "/workspace/example_service_cloud_run/app.py", + "process": 2, + "processName": "MainProcess", + "relativeCreated": 5152.963876724243, + "stack_info": null, + "thread": 68328473233152, + "threadName": "ThreadPoolExecutor-0_2" + } + }, + "attributes": { + "message_number": "1", + "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", + "sender_type": "CHILD", + "version": "0.51.0" + } + }, + { + "event": { + "kind": "log_record", + "log_record": { + "args": null, + "created": 1709739858.190219, + "exc_info": null, + "exc_text": null, + "filename": "submodule.py", + "funcName": "do_something", + "levelname": "INFO", + "levelno": 20, + "lineno": 8, + "module": "submodule", + "msecs": 190.21892547607422, + "msg": "Submodules are included in logging.", + "name": "example_service_cloud_run.submodule", + "pathname": "/workspace/example_service_cloud_run/submodule.py", + "process": 2, + "processName": "MainProcess", + "relativeCreated": 5155.384063720703, + "stack_info": null, + "thread": 68328473233152, + "threadName": "ThreadPoolExecutor-0_2" + } + }, + "attributes": { + "message_number": "2", + "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", + "sender_type": "CHILD", + "version": "0.51.0" + } + }, + { + "event": { + "kind": "log_record", + "log_record": { + "args": null, + "created": 1709739860.2234063, + "exc_info": null, + "exc_text": null, + "filename": "analysis.py", + "funcName": "finalise", + "levelname": "INFO", + "levelno": 20, + "lineno": 158, + "module": "analysis", + "msecs": 223.40631484985352, + "msg": "Validated output values and output manifest against the twine.", + "name": "octue.resources.analysis", + "pathname": "/usr/local/lib/python3.9/site-packages/octue/resources/analysis.py", + "process": 2, + "processName": "MainProcess", + "relativeCreated": 7188.571453094482, + "stack_info": null, + "thread": 68328473233152, + "threadName": "ThreadPoolExecutor-0_2" + } + }, + "attributes": { + "message_number": "3", + "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", + "sender_type": "CHILD", + "version": "0.51.0" + } + }, + { + "event": { + "kind": "log_record", + "log_record": { + "args": null, + "created": 1709739861.5923417, + "exc_info": null, + "exc_text": null, + "filename": "analysis.py", + "funcName": "finalise", + "levelname": "INFO", + "levelno": 20, + "lineno": 174, + "module": "analysis", + "msecs": 592.3416614532471, + "msg": "Uploaded output datasets to 'gs://octue-sdk-python-test-bucket/example_output_datasets'.", + "name": "octue.resources.analysis", + "pathname": "/usr/local/lib/python3.9/site-packages/octue/resources/analysis.py", + "process": 2, + "processName": "MainProcess", + "relativeCreated": 8557.506799697876, + "stack_info": null, + "thread": 68328473233152, + "threadName": "ThreadPoolExecutor-0_2" + } + }, + "attributes": { + "message_number": "4", + "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", + "sender_type": "CHILD", + "version": "0.51.0" + } + }, + { + "event": { + "kind": "log_record", + "log_record": { + "args": null, + "created": 1709739861.5949728, + "exc_info": null, + "exc_text": null, + "filename": "app.py", + "funcName": "run", + "levelname": "INFO", + "levelno": 20, + "lineno": 28, + "module": "app", + "msecs": 594.9728488922119, + "msg": "Finished example analysis.", + "name": "app", + "pathname": "/workspace/example_service_cloud_run/app.py", + "process": 2, + "processName": "MainProcess", + "relativeCreated": 8560.13798713684, + "stack_info": null, + "thread": 68328473233152, + "threadName": "ThreadPoolExecutor-0_2" + } + }, + "attributes": { + "message_number": "5", + "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", + "sender_type": "CHILD", + "version": "0.51.0" + } + }, + { + "event": { + "kind": "result", + "output_manifest": { + "datasets": { + "example_dataset": { + "files": [ + "gs://octue-sdk-python-test-bucket/example_output_datasets/example_dataset/output.dat" + ], + "id": "419bff6b-08c3-4c16-9eb1-5d1709168003", + "labels": [], + "name": "divergent-strange-gharial-of-pizza", + "path": "https://storage.googleapis.com/octue-sdk-python-test-bucket/example_output_datasets/example_dataset/.signed_metadata_files/divergent-strange-gharial-of-pizza", + "tags": {} + } + }, + "id": "a13713ae-f207-41c6-9e29-0a848ced6039", + "name": null + }, + "output_values": [1, 2, 3, 4, 5] + }, + "attributes": { + "message_number": "6", + "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", + "sender_type": "CHILD", + "version": "0.51.0" + } + }, + { + "event": { + "datetime": "2024-03-06T15:46:18.167424", + "kind": "heartbeat" + }, + "attributes": { + "message_number": "7", + "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", + "sender_type": "CHILD", + "version": "0.51.0" + } + } +] From 69997b58bb547448d39ecd8f0b846251f33bdff5 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 6 Mar 2024 17:36:55 +0000 Subject: [PATCH 033/169] REF: Rename `event_handler` module to `handler` --- octue/cloud/events/{event_handler.py => handler.py} | 0 octue/cloud/events/replayer.py | 2 +- octue/cloud/pub_sub/message_handler.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename octue/cloud/events/{event_handler.py => handler.py} (100%) diff --git a/octue/cloud/events/event_handler.py b/octue/cloud/events/handler.py similarity index 100% rename from octue/cloud/events/event_handler.py rename to octue/cloud/events/handler.py diff --git a/octue/cloud/events/replayer.py b/octue/cloud/events/replayer.py index f405f8c29..b21175fd1 100644 --- a/octue/cloud/events/replayer.py +++ b/octue/cloud/events/replayer.py @@ -1,6 +1,6 @@ import logging -from octue.cloud.events.event_handler import EventHandler +from octue.cloud.events.handler import EventHandler from octue.cloud.events.validation import SERVICE_COMMUNICATION_SCHEMA from octue.cloud.pub_sub.service import Service from octue.resources.service_backends import ServiceBackend diff --git a/octue/cloud/pub_sub/message_handler.py b/octue/cloud/pub_sub/message_handler.py index c56a44702..c9673bc26 100644 --- a/octue/cloud/pub_sub/message_handler.py +++ b/octue/cloud/pub_sub/message_handler.py @@ -6,7 +6,7 @@ from google.api_core import retry from google.cloud.pubsub_v1 import SubscriberClient -from octue.cloud.events.event_handler import EventHandler +from octue.cloud.events.handler import EventHandler from octue.cloud.events.validation import SERVICE_COMMUNICATION_SCHEMA from octue.cloud.pub_sub.events import extract_event_and_attributes_from_pub_sub from octue.utils.threads import RepeatingTimer From 35045838c6b63ee270af5c563d68f01ebf510b12 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 6 Mar 2024 17:37:51 +0000 Subject: [PATCH 034/169] REF: Rename `EventHandler` to `AbstractEventHandler` --- octue/cloud/events/handler.py | 2 +- octue/cloud/events/replayer.py | 4 ++-- octue/cloud/pub_sub/message_handler.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/octue/cloud/events/handler.py b/octue/cloud/events/handler.py index f5674c5fa..663626141 100644 --- a/octue/cloud/events/handler.py +++ b/octue/cloud/events/handler.py @@ -27,7 +27,7 @@ PARENT_SDK_VERSION = importlib.metadata.version("octue") -class EventHandler: +class AbstractEventHandler: question_uuid: str def __init__( diff --git a/octue/cloud/events/replayer.py b/octue/cloud/events/replayer.py index b21175fd1..1adf7ebf7 100644 --- a/octue/cloud/events/replayer.py +++ b/octue/cloud/events/replayer.py @@ -1,6 +1,6 @@ import logging -from octue.cloud.events.handler import EventHandler +from octue.cloud.events.handler import AbstractEventHandler from octue.cloud.events.validation import SERVICE_COMMUNICATION_SCHEMA from octue.cloud.pub_sub.service import Service from octue.resources.service_backends import ServiceBackend @@ -9,7 +9,7 @@ logger = logging.getLogger(__name__) -class EventReplayer(EventHandler): +class EventReplayer(AbstractEventHandler): def __init__( self, receiving_service=None, diff --git a/octue/cloud/pub_sub/message_handler.py b/octue/cloud/pub_sub/message_handler.py index c9673bc26..c25bd00d0 100644 --- a/octue/cloud/pub_sub/message_handler.py +++ b/octue/cloud/pub_sub/message_handler.py @@ -6,7 +6,7 @@ from google.api_core import retry from google.cloud.pubsub_v1 import SubscriberClient -from octue.cloud.events.handler import EventHandler +from octue.cloud.events.handler import AbstractEventHandler from octue.cloud.events.validation import SERVICE_COMMUNICATION_SCHEMA from octue.cloud.pub_sub.events import extract_event_and_attributes_from_pub_sub from octue.utils.threads import RepeatingTimer @@ -19,7 +19,7 @@ PARENT_SDK_VERSION = importlib.metadata.version("octue") -class OrderedMessageHandler(EventHandler): +class OrderedMessageHandler(AbstractEventHandler): """A handler for Google Pub/Sub messages received via a pull subscription that ensures messages are handled in the order they were sent. From 533065a9ebcc92325f7399f9ad9ce787cb20a4fd Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 6 Mar 2024 17:47:11 +0000 Subject: [PATCH 035/169] REF: Rename `OrderedMessageHandler` to `PubSubEventHandler` --- octue/cloud/emulators/child.py | 2 +- .../{message_handler.py => event_handler.py} | 2 +- octue/cloud/pub_sub/service.py | 12 ++-- tests/cloud/pub_sub/test_message_handler.py | 72 +++++++++---------- 4 files changed, 44 insertions(+), 44 deletions(-) rename octue/cloud/pub_sub/{message_handler.py => event_handler.py} (99%) diff --git a/octue/cloud/emulators/child.py b/octue/cloud/emulators/child.py index daec2d2b0..8d4ab3b10 100644 --- a/octue/cloud/emulators/child.py +++ b/octue/cloud/emulators/child.py @@ -322,7 +322,7 @@ def __init__(self): patches=[ patch("octue.cloud.pub_sub.service.Topic", new=MockTopic), patch("octue.cloud.pub_sub.service.Subscription", new=MockSubscription), - patch("octue.cloud.pub_sub.message_handler.SubscriberClient", new=MockSubscriber), + patch("octue.cloud.pub_sub.event_handler.SubscriberClient", new=MockSubscriber), patch("google.cloud.pubsub_v1.SubscriberClient", new=MockSubscriber), ] ) diff --git a/octue/cloud/pub_sub/message_handler.py b/octue/cloud/pub_sub/event_handler.py similarity index 99% rename from octue/cloud/pub_sub/message_handler.py rename to octue/cloud/pub_sub/event_handler.py index c25bd00d0..a92b735e1 100644 --- a/octue/cloud/pub_sub/message_handler.py +++ b/octue/cloud/pub_sub/event_handler.py @@ -19,7 +19,7 @@ PARENT_SDK_VERSION = importlib.metadata.version("octue") -class OrderedMessageHandler(AbstractEventHandler): +class PubSubEventHandler(AbstractEventHandler): """A handler for Google Pub/Sub messages received via a pull subscription that ensures messages are handled in the order they were sent. diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index d49e5b2e8..af6538db9 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -16,9 +16,9 @@ import octue.exceptions from octue.cloud.events.validation import raise_if_event_is_invalid from octue.cloud.pub_sub import Subscription, Topic +from octue.cloud.pub_sub.event_handler import PubSubEventHandler from octue.cloud.pub_sub.events import extract_event_and_attributes_from_pub_sub from octue.cloud.pub_sub.logging import GooglePubSubHandler -from octue.cloud.pub_sub.message_handler import OrderedMessageHandler from octue.cloud.service_id import ( convert_service_id_to_pub_sub_form, create_sruid, @@ -93,7 +93,7 @@ def __init__(self, backend, service_id=None, run_function=None, name=None, servi self._pub_sub_id = convert_service_id_to_pub_sub_form(self.id) self._local_sdk_version = importlib.metadata.version("octue") self._publisher = None - self._message_handler = None + self._event_handler = None def __repr__(self): """Represent the service as a string. @@ -123,8 +123,8 @@ def received_messages(self): :return list(dict)|None: """ - if self._message_handler: - return self._message_handler.handled_messages + if self._event_handler: + return self._event_handler.handled_messages return None def serve(self, timeout=None, delete_topic_and_subscription_on_exit=False, allow_existing=False, detach=False): @@ -391,7 +391,7 @@ def wait_for_answer( service_name = get_sruid_from_pub_sub_resource_name(subscription.name) - self._message_handler = OrderedMessageHandler( + self._event_handler = PubSubEventHandler( subscription=subscription, receiving_service=self, handle_monitor_message=handle_monitor_message, @@ -400,7 +400,7 @@ def wait_for_answer( ) try: - return self._message_handler.handle_messages( + return self._event_handler.handle_messages( timeout=timeout, maximum_heartbeat_interval=maximum_heartbeat_interval, ) diff --git a/tests/cloud/pub_sub/test_message_handler.py b/tests/cloud/pub_sub/test_message_handler.py index 8855fd467..39327779f 100644 --- a/tests/cloud/pub_sub/test_message_handler.py +++ b/tests/cloud/pub_sub/test_message_handler.py @@ -12,7 +12,7 @@ MockTopic, ) from octue.cloud.emulators.child import ServicePatcher -from octue.cloud.pub_sub.message_handler import OrderedMessageHandler +from octue.cloud.pub_sub.event_handler import PubSubEventHandler from octue.resources.service_backends import GCPPubSubBackend from tests import TEST_PROJECT_NAME from tests.base import BaseTestCase @@ -34,13 +34,13 @@ def create_mock_topic_and_subscription(): return question_uuid, topic, subscription -class TestOrderedMessageHandler(BaseTestCase): +class TestPubSubEventHandler(BaseTestCase): def test_timeout(self): """Test that a TimeoutError is raised if message handling takes longer than the given timeout.""" question_uuid, _, mock_subscription = create_mock_topic_and_subscription() - with patch("octue.cloud.pub_sub.message_handler.SubscriberClient", MockSubscriber): - message_handler = OrderedMessageHandler( + with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): + message_handler = PubSubEventHandler( subscription=mock_subscription, receiving_service=parent, message_handlers={"test": lambda message: None, "finish-test": lambda message: message}, @@ -54,8 +54,8 @@ def test_in_order_messages_are_handled_in_order(self): """Test that messages received in order are handled in order.""" question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() - with patch("octue.cloud.pub_sub.message_handler.SubscriberClient", MockSubscriber): - message_handler = OrderedMessageHandler( + with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): + message_handler = PubSubEventHandler( subscription=mock_subscription, receiving_service=parent, message_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, @@ -86,8 +86,8 @@ def test_out_of_order_messages_are_handled_in_order(self): """Test that messages received out of order are handled in order.""" question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() - with patch("octue.cloud.pub_sub.message_handler.SubscriberClient", MockSubscriber): - message_handler = OrderedMessageHandler( + with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): + message_handler = PubSubEventHandler( subscription=mock_subscription, receiving_service=parent, message_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, @@ -139,8 +139,8 @@ def test_out_of_order_messages_with_end_message_first_are_handled_in_order(self) """ question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() - with patch("octue.cloud.pub_sub.message_handler.SubscriberClient", MockSubscriber): - message_handler = OrderedMessageHandler( + with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): + message_handler = PubSubEventHandler( subscription=mock_subscription, receiving_service=parent, message_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, @@ -190,8 +190,8 @@ def test_no_timeout(self): """Test that message handling works with no timeout.""" question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() - with patch("octue.cloud.pub_sub.message_handler.SubscriberClient", MockSubscriber): - message_handler = OrderedMessageHandler( + with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): + message_handler = PubSubEventHandler( subscription=mock_subscription, receiving_service=parent, message_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, @@ -230,8 +230,8 @@ def test_delivery_acknowledgement(self): """Test that a delivery acknowledgement message is handled correctly.""" question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() - with patch("octue.cloud.pub_sub.message_handler.SubscriberClient", MockSubscriber): - message_handler = OrderedMessageHandler(subscription=mock_subscription, receiving_service=parent) + with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): + message_handler = PubSubEventHandler(subscription=mock_subscription, receiving_service=parent) child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) @@ -256,8 +256,8 @@ def test_error_raised_if_heartbeat_not_received_before_checked(self): """Test that an error is raised if a heartbeat isn't received before a heartbeat is first checked for.""" question_uuid, _, mock_subscription = create_mock_topic_and_subscription() - with patch("octue.cloud.pub_sub.message_handler.SubscriberClient", MockSubscriber): - message_handler = OrderedMessageHandler(subscription=mock_subscription, receiving_service=parent) + with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): + message_handler = PubSubEventHandler(subscription=mock_subscription, receiving_service=parent) with self.assertRaises(TimeoutError) as error: message_handler.handle_messages(maximum_heartbeat_interval=0) @@ -269,8 +269,8 @@ def test_error_raised_if_heartbeats_stop_being_received(self): """Test that an error is raised if heartbeats stop being received within the maximum interval.""" question_uuid, _, mock_subscription = create_mock_topic_and_subscription() - with patch("octue.cloud.pub_sub.message_handler.SubscriberClient", MockSubscriber): - message_handler = OrderedMessageHandler(subscription=mock_subscription, receiving_service=parent) + with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): + message_handler = PubSubEventHandler(subscription=mock_subscription, receiving_service=parent) message_handler._last_heartbeat = datetime.datetime.now() - datetime.timedelta(seconds=30) @@ -283,8 +283,8 @@ def test_error_not_raised_if_heartbeat_has_been_received_in_maximum_allowed_inte """Test that an error is not raised if a heartbeat has been received in the maximum allowed interval.""" question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() - with patch("octue.cloud.pub_sub.message_handler.SubscriberClient", MockSubscriber): - message_handler = OrderedMessageHandler(subscription=mock_subscription, receiving_service=parent) + with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): + message_handler = PubSubEventHandler(subscription=mock_subscription, receiving_service=parent) message_handler._last_heartbeat = datetime.datetime.now() @@ -308,7 +308,7 @@ def test_error_not_raised_if_heartbeat_has_been_received_in_maximum_allowed_inte child._send_message(message=message["event"], attributes=message["attributes"], topic=mock_topic) with patch( - "octue.cloud.pub_sub.message_handler.OrderedMessageHandler._time_since_last_heartbeat", + "octue.cloud.pub_sub.event_handler.PubSubEventHandler._time_since_last_heartbeat", datetime.timedelta(seconds=0), ): message_handler.handle_messages(maximum_heartbeat_interval=0) @@ -317,8 +317,8 @@ def test_time_since_last_heartbeat_is_none_if_no_heartbeat_received_yet(self): """Test that the time since the last heartbeat is `None` if no heartbeat has been received yet.""" question_uuid, _, mock_subscription = create_mock_topic_and_subscription() - with patch("octue.cloud.pub_sub.message_handler.SubscriberClient", MockSubscriber): - message_handler = OrderedMessageHandler(subscription=mock_subscription, receiving_service=parent) + with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): + message_handler = PubSubEventHandler(subscription=mock_subscription, receiving_service=parent) self.assertIsNone(message_handler._time_since_last_heartbeat) @@ -327,15 +327,15 @@ def test_total_run_time_is_none_if_handle_messages_has_not_been_called(self): called. """ question_uuid, _, mock_subscription = create_mock_topic_and_subscription() - message_handler = OrderedMessageHandler(subscription=mock_subscription, receiving_service=parent) + message_handler = PubSubEventHandler(subscription=mock_subscription, receiving_service=parent) self.assertIsNone(message_handler.total_run_time) def test_time_since_missing_message_is_none_if_no_unhandled_missing_messages(self): - """Test that the `OrderedMessageHandler.time_since_missing_message` property is `None` if there are no unhandled + """Test that the `PubSubEventHandler.time_since_missing_message` property is `None` if there are no unhandled missing messages. """ question_uuid, _, mock_subscription = create_mock_topic_and_subscription() - message_handler = OrderedMessageHandler(subscription=mock_subscription, receiving_service=parent) + message_handler = PubSubEventHandler(subscription=mock_subscription, receiving_service=parent) self.assertIsNone(message_handler.time_since_missing_message) def test_missing_messages_at_start_can_be_skipped(self): @@ -344,8 +344,8 @@ def test_missing_messages_at_start_can_be_skipped(self): """ question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() - with patch("octue.cloud.pub_sub.message_handler.SubscriberClient", MockSubscriber): - message_handler = OrderedMessageHandler( + with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): + message_handler = PubSubEventHandler( subscription=mock_subscription, receiving_service=parent, message_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, @@ -397,8 +397,8 @@ def test_missing_messages_in_middle_can_skipped(self): """Test that missing messages in the middle of the event stream can be skipped.""" question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() - with patch("octue.cloud.pub_sub.message_handler.SubscriberClient", MockSubscriber): - message_handler = OrderedMessageHandler( + with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): + message_handler = PubSubEventHandler( subscription=mock_subscription, receiving_service=parent, message_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, @@ -454,8 +454,8 @@ def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): """Test that multiple blocks of missing messages in the middle of the event stream can be skipped.""" question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() - with patch("octue.cloud.pub_sub.message_handler.SubscriberClient", MockSubscriber): - message_handler = OrderedMessageHandler( + with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): + message_handler = PubSubEventHandler( subscription=mock_subscription, receiving_service=parent, message_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, @@ -541,8 +541,8 @@ def test_all_messages_missing_apart_from_result(self): """Test that the result message is still handled if all other messages are missing.""" question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() - with patch("octue.cloud.pub_sub.message_handler.SubscriberClient", MockSubscriber): - message_handler = OrderedMessageHandler( + with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): + message_handler = PubSubEventHandler( subscription=mock_subscription, receiving_service=parent, message_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, @@ -579,7 +579,7 @@ def test_pull_and_enqueue_available_messages(self): topic=mock_topic, ) - message_handler = OrderedMessageHandler( + message_handler = PubSubEventHandler( subscription=mock_subscription, receiving_service=parent, message_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, @@ -618,7 +618,7 @@ def test_timeout_error_raised_if_result_message_not_received_in_time(self): topic=mock_topic, ) - message_handler = OrderedMessageHandler( + message_handler = PubSubEventHandler( subscription=mock_subscription, receiving_service=parent, message_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, From 585e0a8e0a07675f30b77527633a47d0995f90e0 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 6 Mar 2024 17:50:28 +0000 Subject: [PATCH 036/169] REF: Rename `handle_messages` method to `handle_events` skipci --- octue/cloud/events/handler.py | 4 +++ octue/cloud/pub_sub/event_handler.py | 6 ++--- octue/cloud/pub_sub/service.py | 2 +- tests/cloud/pub_sub/test_message_handler.py | 30 ++++++++++----------- 4 files changed, 23 insertions(+), 19 deletions(-) diff --git a/octue/cloud/events/handler.py b/octue/cloud/events/handler.py index 663626141..103135dbf 100644 --- a/octue/cloud/events/handler.py +++ b/octue/cloud/events/handler.py @@ -78,6 +78,10 @@ def time_since_missing_message(self): return time.perf_counter() - self._missing_message_detection_time + @abc.abstractmethod + def handle_events(self, *args, **kwargs): + pass + @abc.abstractmethod def _extract_event_and_attributes(self, event): pass diff --git a/octue/cloud/pub_sub/event_handler.py b/octue/cloud/pub_sub/event_handler.py index a92b735e1..4bcac4119 100644 --- a/octue/cloud/pub_sub/event_handler.py +++ b/octue/cloud/pub_sub/event_handler.py @@ -67,10 +67,10 @@ def __init__( @property def total_run_time(self): - """Get the amount of time elapsed since `self.handle_messages` was called. If it hasn't been called yet, it will + """Get the amount of time elapsed since `self.handle_events` was called. If it hasn't been called yet, it will be `None`. - :return float|None: the amount of time since `self.handle_messages` was called (in seconds) + :return float|None: the amount of time since `self.handle_events` was called (in seconds) """ if self._start_time is None: return None @@ -88,7 +88,7 @@ def _time_since_last_heartbeat(self): return datetime.now() - self._last_heartbeat - def handle_messages(self, timeout=60, maximum_heartbeat_interval=300): + def handle_events(self, timeout=60, maximum_heartbeat_interval=300): """Pull messages and handle them in the order they were sent until a result is returned by a message handler, then return that result. diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index af6538db9..06be5520f 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -400,7 +400,7 @@ def wait_for_answer( ) try: - return self._event_handler.handle_messages( + return self._event_handler.handle_events( timeout=timeout, maximum_heartbeat_interval=maximum_heartbeat_interval, ) diff --git a/tests/cloud/pub_sub/test_message_handler.py b/tests/cloud/pub_sub/test_message_handler.py index 39327779f..e88c3537b 100644 --- a/tests/cloud/pub_sub/test_message_handler.py +++ b/tests/cloud/pub_sub/test_message_handler.py @@ -48,7 +48,7 @@ def test_timeout(self): ) with self.assertRaises(TimeoutError): - message_handler.handle_messages(timeout=0) + message_handler.handle_events(timeout=0) def test_in_order_messages_are_handled_in_order(self): """Test that messages received in order are handled in order.""" @@ -74,7 +74,7 @@ def test_in_order_messages_are_handled_in_order(self): for message in messages: child._send_message(message=message["event"], attributes=message["attributes"], topic=mock_topic) - result = message_handler.handle_messages() + result = message_handler.handle_events() self.assertEqual(result, "This is the result.") self.assertEqual( @@ -119,7 +119,7 @@ def test_out_of_order_messages_are_handled_in_order(self): mock_topic.messages_published = message["event"]["order"] child._send_message(message=message["event"], attributes=message["attributes"], topic=mock_topic) - result = message_handler.handle_messages() + result = message_handler.handle_events() self.assertEqual(result, "This is the result.") @@ -172,7 +172,7 @@ def test_out_of_order_messages_with_end_message_first_are_handled_in_order(self) mock_topic.messages_published = message["event"]["order"] child._send_message(message=message["event"], attributes=message["attributes"], topic=mock_topic) - result = message_handler.handle_messages() + result = message_handler.handle_events() self.assertEqual(result, "This is the result.") @@ -218,7 +218,7 @@ def test_no_timeout(self): for message in messages: child._send_message(message=message["event"], attributes=message["attributes"], topic=mock_topic) - result = message_handler.handle_messages(timeout=None) + result = message_handler.handle_events(timeout=None) self.assertEqual(result, "This is the result.") self.assertEqual( @@ -249,7 +249,7 @@ def test_delivery_acknowledgement(self): for message in messages: child._send_message(message=message["event"], attributes=message["attributes"], topic=mock_topic) - result = message_handler.handle_messages() + result = message_handler.handle_events() self.assertEqual(result, {"output_values": None, "output_manifest": None}) def test_error_raised_if_heartbeat_not_received_before_checked(self): @@ -260,7 +260,7 @@ def test_error_raised_if_heartbeat_not_received_before_checked(self): message_handler = PubSubEventHandler(subscription=mock_subscription, receiving_service=parent) with self.assertRaises(TimeoutError) as error: - message_handler.handle_messages(maximum_heartbeat_interval=0) + message_handler.handle_events(maximum_heartbeat_interval=0) # Check that the timeout is due to a heartbeat not being received. self.assertIn("heartbeat", error.exception.args[0]) @@ -275,7 +275,7 @@ def test_error_raised_if_heartbeats_stop_being_received(self): message_handler._last_heartbeat = datetime.datetime.now() - datetime.timedelta(seconds=30) with self.assertRaises(TimeoutError) as error: - message_handler.handle_messages(maximum_heartbeat_interval=0) + message_handler.handle_events(maximum_heartbeat_interval=0) self.assertIn("heartbeat", error.exception.args[0]) @@ -311,7 +311,7 @@ def test_error_not_raised_if_heartbeat_has_been_received_in_maximum_allowed_inte "octue.cloud.pub_sub.event_handler.PubSubEventHandler._time_since_last_heartbeat", datetime.timedelta(seconds=0), ): - message_handler.handle_messages(maximum_heartbeat_interval=0) + message_handler.handle_events(maximum_heartbeat_interval=0) def test_time_since_last_heartbeat_is_none_if_no_heartbeat_received_yet(self): """Test that the time since the last heartbeat is `None` if no heartbeat has been received yet.""" @@ -322,8 +322,8 @@ def test_time_since_last_heartbeat_is_none_if_no_heartbeat_received_yet(self): self.assertIsNone(message_handler._time_since_last_heartbeat) - def test_total_run_time_is_none_if_handle_messages_has_not_been_called(self): - """Test that the total run time for the message handler is `None` if the `handle_messages` method has not been + def test_total_run_time_is_none_if_handle_events_has_not_been_called(self): + """Test that the total run time for the message handler is `None` if the `handle_events` method has not been called. """ question_uuid, _, mock_subscription = create_mock_topic_and_subscription() @@ -380,7 +380,7 @@ def test_missing_messages_at_start_can_be_skipped(self): for message in messages: child._send_message(message=message["event"], attributes=message["attributes"], topic=mock_topic) - result = message_handler.handle_messages() + result = message_handler.handle_events() self.assertEqual(result, "This is the result.") self.assertEqual( @@ -437,7 +437,7 @@ def test_missing_messages_in_middle_can_skipped(self): topic=mock_topic, ) - message_handler.handle_messages() + message_handler.handle_events() # Check that all the non-missing messages were handled. self.assertEqual( @@ -520,7 +520,7 @@ def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): for message in messages: child._send_message(message=message["event"], attributes=message["attributes"], topic=mock_topic) - message_handler.handle_messages() + message_handler.handle_events() # Check that all the non-missing messages were handled. self.assertEqual( @@ -562,7 +562,7 @@ def test_all_messages_missing_apart_from_result(self): topic=mock_topic, ) - message_handler.handle_messages() + message_handler.handle_events() # Check that the result message was handled. self.assertEqual(message_handler.handled_messages, [{"kind": "finish-test", "order": 1000}]) From 9d25f0a726f7c6de36fd573709ca37c37739a19d Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 6 Mar 2024 18:19:22 +0000 Subject: [PATCH 037/169] REF: Rename "message" to "event" in event handler classes --- octue/cloud/events/handler.py | 174 ++++++++++---------- octue/cloud/events/replayer.py | 18 +- octue/cloud/pub_sub/event_handler.py | 59 ++++--- octue/cloud/pub_sub/service.py | 4 +- tests/cloud/pub_sub/test_message_handler.py | 64 +++---- 5 files changed, 159 insertions(+), 160 deletions(-) diff --git a/octue/cloud/events/handler.py b/octue/cloud/events/handler.py index 103135dbf..1e23e0a80 100644 --- a/octue/cloud/events/handler.py +++ b/octue/cloud/events/handler.py @@ -34,28 +34,28 @@ def __init__( self, receiving_service, handle_monitor_message=None, - record_messages=True, + record_events=True, service_name="REMOTE", - message_handlers=None, + event_handlers=None, schema=SERVICE_COMMUNICATION_SCHEMA, - skip_missing_messages_after=10, + skip_missing_events_after=10, ): self.receiving_service = receiving_service self.handle_monitor_message = handle_monitor_message - self.record_messages = record_messages + self.record_events = record_events self.service_name = service_name self.schema = schema - self.waiting_messages = None - self.handled_messages = [] - self._previous_message_number = -1 + self.waiting_events = None + self.handled_events = [] + self._previous_event_number = -1 self._child_sdk_version = None - self.skip_missing_messages_after = skip_missing_messages_after - self._missing_message_detection_time = None - self._earliest_waiting_message_number = math.inf + self.skip_missing_events_after = skip_missing_events_after + self._missing_event_detection_time = None + self._earliest_waiting_event_number = math.inf - self._message_handlers = message_handlers or { + self._event_handlers = event_handlers or { "delivery_acknowledgement": self._handle_delivery_acknowledgement, "heartbeat": self._handle_heartbeat, "monitor_message": self._handle_monitor_message, @@ -67,16 +67,16 @@ def __init__( self._log_message_colours = [COLOUR_PALETTE[1], *COLOUR_PALETTE[3:]] @property - def time_since_missing_message(self): - """Get the amount of time elapsed since the last missing message was detected. If no missing messages have been + def time_since_missing_event(self): + """Get the amount of time elapsed since the last missing event was detected. If no missing events have been detected or they've already been skipped past, `None` is returned. :return float|None: """ - if self._missing_message_detection_time is None: + if self._missing_event_detection_time is None: return None - return time.perf_counter() - self._missing_message_detection_time + return time.perf_counter() - self._missing_event_detection_time @abc.abstractmethod def handle_events(self, *args, **kwargs): @@ -87,12 +87,12 @@ def _extract_event_and_attributes(self, event): pass def _extract_and_enqueue_event(self, event): - """Extract an event from the Pub/Sub message and add it to `self.waiting_messages`. + """Extract an event from the Pub/Sub message and add it to `self.waiting_events`. :param dict event: :return None: """ - logger.debug("%r received a message related to question %r.", self.receiving_service, self.question_uuid) + logger.debug("%r received an event related to question %r.", self.receiving_service, self.question_uuid) event, attributes = self._extract_event_and_attributes(event) if not is_event_valid( @@ -105,137 +105,137 @@ def _extract_and_enqueue_event(self, event): ): return - # Get the child's Octue SDK version from the first message. + # Get the child's Octue SDK version from the first event. if not self._child_sdk_version: self._child_sdk_version = attributes["version"] - message_number = attributes["message_number"] + event_number = attributes["message_number"] - if message_number in self.waiting_messages: + if event_number in self.waiting_events: logger.warning( - "%r: Message with duplicate message number %d received for question %s - overwriting original message.", + "%r: Event with duplicate event number %d received for question %s - overwriting original event.", self.receiving_service, - message_number, + event_number, self.question_uuid, ) - self.waiting_messages[message_number] = event + self.waiting_events[event_number] = event - def _attempt_to_handle_waiting_messages(self): - """Attempt to handle messages waiting in `self.waiting_messages`. If these messages aren't consecutive to the - last handled message (i.e. if messages have been received out of order and the next in-order message hasn't been - received yet), just return. After the missing message wait time has passed, if this set of missing messages - haven't arrived but subsequent ones have, skip to the earliest waiting message and continue from there. + def _attempt_to_handle_waiting_events(self): + """Attempt to handle events waiting in `self.waiting_events`. If these events aren't consecutive to the + last handled event (i.e. if events have been received out of order and the next in-order event hasn't been + received yet), just return. After the missing event wait time has passed, if this set of missing events + haven't arrived but subsequent ones have, skip to the earliest waiting event and continue from there. - :return any|None: either a non-`None` result from a message handler or `None` if nothing was returned by the message handlers or if the next in-order message hasn't been received yet + :return any|None: either a non-`None` result from a event handler or `None` if nothing was returned by the event handlers or if the next in-order event hasn't been received yet """ - while self.waiting_messages: + while self.waiting_events: try: - # If the next consecutive message has been received: - message = self.waiting_messages.pop(self._previous_message_number + 1) + # If the next consecutive event has been received: + event = self.waiting_events.pop(self._previous_event_number + 1) - # If the next consecutive message hasn't been received: + # If the next consecutive event hasn't been received: except KeyError: - # Start the missing message timer if it isn't already running. - if self._missing_message_detection_time is None: - self._missing_message_detection_time = time.perf_counter() + # Start the missing event timer if it isn't already running. + if self._missing_event_detection_time is None: + self._missing_event_detection_time = time.perf_counter() - if self.time_since_missing_message > self.skip_missing_messages_after: - message = self._skip_to_earliest_waiting_message() + if self.time_since_missing_event > self.skip_missing_events_after: + event = self._skip_to_earliest_waiting_event() - # Declare there are no more missing messages. - self._missing_message_detection_time = None + # Declare there are no more missing events. + self._missing_event_detection_time = None - if not message: + if not event: return else: return - result = self._handle_message(message) + result = self._handle_event(event) if result is not None: return result - def _skip_to_earliest_waiting_message(self): - """Get the earliest waiting message and set the message handler up to continue from it. + def _skip_to_earliest_waiting_event(self): + """Get the earliest waiting event and set the event handler up to continue from it. :return dict|None: """ try: - message = self.waiting_messages.pop(self._earliest_waiting_message_number) + event = self.waiting_events.pop(self._earliest_waiting_event_number) except KeyError: return - number_of_missing_messages = self._earliest_waiting_message_number - self._previous_message_number - 1 + number_of_missing_events = self._earliest_waiting_event_number - self._previous_event_number - 1 - # Let the message handler know it can handle the next earliest message. - self._previous_message_number = self._earliest_waiting_message_number - 1 + # Let the event handler know it can handle the next earliest event. + self._previous_event_number = self._earliest_waiting_event_number - 1 logger.warning( - "%r: %d consecutive messages missing for question %r after %ds - skipping to next earliest waiting message " - "(message %d).", + "%r: %d consecutive events missing for question %r after %ds - skipping to next earliest waiting event " + "(event %d).", self.receiving_service, - number_of_missing_messages, + number_of_missing_events, self.question_uuid, - self.skip_missing_messages_after, - self._earliest_waiting_message_number, + self.skip_missing_events_after, + self._earliest_waiting_event_number, ) - return message + return event - def _handle_message(self, message): - """Pass a message to its handler and update the previous message number. + def _handle_event(self, event): + """Pass an event to its handler and update the previous event number. - :param dict message: + :param dict event: :return dict|None: """ - self._previous_message_number += 1 + self._previous_event_number += 1 - if self.record_messages: - self.handled_messages.append(message) + if self.record_events: + self.handled_events.append(event) - handler = self._message_handlers[message["kind"]] - return handler(message) + handler = self._event_handlers[event["kind"]] + return handler(event) - def _handle_delivery_acknowledgement(self, message): + def _handle_delivery_acknowledgement(self, event): """Mark the question as delivered to prevent resending it. - :param dict message: + :param dict event: :return None: """ - logger.info("%r's question was delivered at %s.", self.receiving_service, message["datetime"]) + logger.info("%r's question was delivered at %s.", self.receiving_service, event["datetime"]) - def _handle_heartbeat(self, message): + def _handle_heartbeat(self, event): """Record the time the heartbeat was received. - :param dict message: + :param dict event: :return None: """ self._last_heartbeat = datetime.now() logger.info("Heartbeat received from service %r for question %r.", self.service_name, self.question_uuid) - def _handle_monitor_message(self, message): + def _handle_monitor_message(self, event): """Send a monitor message to the handler if one has been provided. - :param dict message: + :param dict event: :return None: """ logger.debug("%r received a monitor message.", self.receiving_service) if self.handle_monitor_message is not None: - self.handle_monitor_message(message["data"]) + self.handle_monitor_message(event["data"]) - def _handle_log_message(self, message): - """Deserialise the message into a log record and pass it to the local log handlers, adding [] to + def _handle_log_message(self, event): + """Deserialise the event into a log record and pass it to the local log handlers, adding [] to the start of the log message. - :param dict message: + :param dict event: :return None: """ - record = logging.makeLogRecord(message["log_record"]) + record = logging.makeLogRecord(event["log_record"]) - # Add information about the immediate child sending the message and colour it with the first colour in the + # Add information about the immediate child sending the event and colour it with the first colour in the # colour palette. immediate_child_analysis_section = colourise( f"[{self.service_name} | analysis-{self.question_uuid}]", @@ -255,41 +255,41 @@ def _handle_log_message(self, message): record.msg = " ".join([immediate_child_analysis_section, *subchild_analysis_sections, final_message]) logger.handle(record) - def _handle_exception(self, message): + def _handle_exception(self, event): """Raise the exception from the responding service that is serialised in `data`. - :param dict message: + :param dict event: :raise Exception: :return None: """ exception_message = "\n\n".join( ( - message["exception_message"], + event["exception_message"], f"The following traceback was captured from the remote service {self.service_name!r}:", - "".join(message["exception_traceback"]), + "".join(event["exception_traceback"]), ) ) try: - exception_type = EXCEPTIONS_MAPPING[message["exception_type"]] + exception_type = EXCEPTIONS_MAPPING[event["exception_type"]] # Allow unknown exception types to still be raised. except KeyError: - exception_type = type(message["exception_type"], (Exception,), {}) + exception_type = type(event["exception_type"], (Exception,), {}) raise exception_type(exception_message) - def _handle_result(self, message): - """Convert the result to the correct form, deserialising the output manifest if it is present in the message. + def _handle_result(self, event): + """Convert the result to the correct form, deserialising the output manifest if it is present in the event. - :param dict message: + :param dict event: :return dict: """ logger.info("%r received an answer to question %r.", self.receiving_service, self.question_uuid) - if message.get("output_manifest"): - output_manifest = Manifest.deserialise(message["output_manifest"]) + if event.get("output_manifest"): + output_manifest = Manifest.deserialise(event["output_manifest"]) else: output_manifest = None - return {"output_values": message.get("output_values"), "output_manifest": output_manifest} + return {"output_values": event.get("output_values"), "output_manifest": output_manifest} diff --git a/octue/cloud/events/replayer.py b/octue/cloud/events/replayer.py index 1adf7ebf7..7d977d4d0 100644 --- a/octue/cloud/events/replayer.py +++ b/octue/cloud/events/replayer.py @@ -14,31 +14,31 @@ def __init__( self, receiving_service=None, handle_monitor_message=None, - record_messages=True, + record_events=True, service_name="REMOTE", - message_handlers=None, + event_handlers=None, schema=SERVICE_COMMUNICATION_SCHEMA, ): super().__init__( receiving_service or Service(backend=ServiceBackend(), service_id="local/local:local"), handle_monitor_message=handle_monitor_message, - record_messages=record_messages, + record_events=record_events, service_name=service_name, - message_handlers=message_handlers, + event_handlers=event_handlers, schema=schema, - skip_missing_messages_after=0, + skip_missing_events_after=0, ) def handle_events(self, events): self.question_uuid = events[0]["attributes"]["question_uuid"] - self.waiting_messages = {} - self._previous_message_number = -1 + self.waiting_events = {} + self._previous_event_number = -1 for event in events: self._extract_and_enqueue_event(event) - self._earliest_waiting_message_number = min(self.waiting_messages.keys()) - return self._attempt_to_handle_waiting_messages() + self._earliest_waiting_event_number = min(self.waiting_events.keys()) + return self._attempt_to_handle_waiting_events() def _extract_event_and_attributes(self, event): event["attributes"]["message_number"] = int(event["attributes"]["message_number"]) diff --git a/octue/cloud/pub_sub/event_handler.py b/octue/cloud/pub_sub/event_handler.py index 4bcac4119..7e9da7070 100644 --- a/octue/cloud/pub_sub/event_handler.py +++ b/octue/cloud/pub_sub/event_handler.py @@ -20,17 +20,16 @@ class PubSubEventHandler(AbstractEventHandler): - """A handler for Google Pub/Sub messages received via a pull subscription that ensures messages are handled in the - order they were sent. + """A handler for events received as Google Pub/Sub messages from a pull subscription. :param octue.cloud.pub_sub.subscription.Subscription subscription: the subscription messages are pulled from - :param octue.cloud.pub_sub.service.Service receiving_service: the service that's receiving the messages + :param octue.cloud.pub_sub.service.Service receiving_service: the service that's receiving the events :param callable|None handle_monitor_message: a function to handle monitor messages (e.g. send them to an endpoint for plotting or displaying) - this function should take a single JSON-compatible python primitive - :param bool record_messages: if `True`, record received messages in the `received_messages` attribute + :param bool record_events: if `True`, record received events in the `received_events` attribute :param str service_name: an arbitrary name to refer to the service subscribed to by (used for labelling its remote log messages) - :param dict|None message_handlers: a mapping of message type names to callables that handle each type of message. The handlers should not mutate the messages. - :param dict|str schema: the JSON schema (or URI of one) to validate messages against - :param int|float skip_missing_messages_after: the number of seconds after which to skip any messages if they haven't arrived but subsequent messages have + :param dict|None event_handlers: a mapping of event type names to callables that handle each type of event. The handlers should not mutate the events. + :param dict|str schema: the JSON schema (or URI of one) to validate events against + :param int|float skip_missing_events_after: the number of seconds after which to skip any events if they haven't arrived but subsequent events have :return None: """ @@ -39,26 +38,26 @@ def __init__( subscription, receiving_service, handle_monitor_message=None, - record_messages=True, + record_events=True, service_name="REMOTE", - message_handlers=None, + event_handlers=None, schema=SERVICE_COMMUNICATION_SCHEMA, - skip_missing_messages_after=10, + skip_missing_events_after=10, ): self.subscription = subscription super().__init__( receiving_service, handle_monitor_message=handle_monitor_message, - record_messages=record_messages, + record_events=record_events, service_name=service_name, - message_handlers=message_handlers, + event_handlers=event_handlers, schema=schema, - skip_missing_messages_after=skip_missing_messages_after, + skip_missing_events_after=skip_missing_events_after, ) self.question_uuid = self.subscription.path.split(".")[-1] - self.waiting_messages = None + self.waiting_events = None self._subscriber = SubscriberClient() self._heartbeat_checker = None self._last_heartbeat = None @@ -89,17 +88,17 @@ def _time_since_last_heartbeat(self): return datetime.now() - self._last_heartbeat def handle_events(self, timeout=60, maximum_heartbeat_interval=300): - """Pull messages and handle them in the order they were sent until a result is returned by a message handler, + """Pull events and handle them in the order they were sent until a result is returned by a event handler, then return that result. :param float|None timeout: how long to wait for an answer before raising a `TimeoutError` :param int|float maximum_heartbeat_interval: the maximum amount of time (in seconds) allowed between child heartbeats before an error is raised - :raise TimeoutError: if the timeout is exceeded before receiving the final message - :return dict: the first result returned by a message handler + :raise TimeoutError: if the timeout is exceeded before receiving the final event + :return dict: the first result returned by a event handler """ self._start_time = time.perf_counter() - self.waiting_messages = {} - self._previous_message_number = -1 + self.waiting_events = {} + self._previous_event_number = -1 self._heartbeat_checker = RepeatingTimer( interval=maximum_heartbeat_interval, @@ -113,8 +112,8 @@ def handle_events(self, timeout=60, maximum_heartbeat_interval=300): while self._alive: pull_timeout = self._check_timeout_and_get_pull_timeout(timeout) - self._pull_and_enqueue_available_messages(timeout=pull_timeout) - result = self._attempt_to_handle_waiting_messages() + self._pull_and_enqueue_available_events(timeout=pull_timeout) + result = self._attempt_to_handle_waiting_events() if result is not None: return result @@ -165,11 +164,11 @@ def _check_timeout_and_get_pull_timeout(self, timeout): return timeout - total_run_time - def _pull_and_enqueue_available_messages(self, timeout): - """Pull as many messages from the subscription as are available and enqueue them in `self.waiting_messages`, + def _pull_and_enqueue_available_events(self, timeout): + """Pull as many events from the subscription as are available and enqueue them in `self.waiting_events`, raising a `TimeoutError` if the timeout is exceeded before succeeding. - :param float|None timeout: how long to wait in seconds for the message before raising a `TimeoutError` + :param float|None timeout: how long to wait in seconds for the event before raising a `TimeoutError` :raise TimeoutError|concurrent.futures.TimeoutError: if the timeout is exceeded :return None: """ @@ -177,7 +176,7 @@ def _pull_and_enqueue_available_messages(self, timeout): attempt = 1 while self._alive: - logger.debug("Pulling messages from Google Pub/Sub: attempt %d.", attempt) + logger.debug("Pulling events from Google Pub/Sub: attempt %d.", attempt) pull_response = self._subscriber.pull( request={"subscription": self.subscription.path, "max_messages": MAX_SIMULTANEOUS_MESSAGES_PULL}, @@ -207,10 +206,10 @@ def _pull_and_enqueue_available_messages(self, timeout): } ) - for message in pull_response.received_messages: - self._extract_and_enqueue_event(message) + for event in pull_response.received_messages: + self._extract_and_enqueue_event(event) - self._earliest_waiting_message_number = min(self.waiting_messages.keys()) + self._earliest_waiting_event_number = min(self.waiting_events.keys()) - def _extract_event_and_attributes(self, message): - return extract_event_and_attributes_from_pub_sub(message.message) + def _extract_event_and_attributes(self, event): + return extract_event_and_attributes_from_pub_sub(event.message) diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index 06be5520f..b4dc1eea8 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -124,7 +124,7 @@ def received_messages(self): :return list(dict)|None: """ if self._event_handler: - return self._event_handler.handled_messages + return self._event_handler.handled_events return None def serve(self, timeout=None, delete_topic_and_subscription_on_exit=False, allow_existing=False, detach=False): @@ -396,7 +396,7 @@ def wait_for_answer( receiving_service=self, handle_monitor_message=handle_monitor_message, service_name=service_name, - record_messages=record_messages, + record_events=record_messages, ) try: diff --git a/tests/cloud/pub_sub/test_message_handler.py b/tests/cloud/pub_sub/test_message_handler.py index e88c3537b..e5ab4f7d4 100644 --- a/tests/cloud/pub_sub/test_message_handler.py +++ b/tests/cloud/pub_sub/test_message_handler.py @@ -43,7 +43,7 @@ def test_timeout(self): message_handler = PubSubEventHandler( subscription=mock_subscription, receiving_service=parent, - message_handlers={"test": lambda message: None, "finish-test": lambda message: message}, + event_handlers={"test": lambda message: None, "finish-test": lambda message: message}, schema={}, ) @@ -58,7 +58,7 @@ def test_in_order_messages_are_handled_in_order(self): message_handler = PubSubEventHandler( subscription=mock_subscription, receiving_service=parent, - message_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, + event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, schema={}, ) @@ -78,7 +78,7 @@ def test_in_order_messages_are_handled_in_order(self): self.assertEqual(result, "This is the result.") self.assertEqual( - message_handler.handled_messages, + message_handler.handled_events, [{"kind": "test"}, {"kind": "test"}, {"kind": "test"}, {"kind": "finish-test"}], ) @@ -90,7 +90,7 @@ def test_out_of_order_messages_are_handled_in_order(self): message_handler = PubSubEventHandler( subscription=mock_subscription, receiving_service=parent, - message_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, + event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, schema={}, ) @@ -124,7 +124,7 @@ def test_out_of_order_messages_are_handled_in_order(self): self.assertEqual(result, "This is the result.") self.assertEqual( - message_handler.handled_messages, + message_handler.handled_events, [ {"kind": "test", "order": 0}, {"kind": "test", "order": 1}, @@ -143,7 +143,7 @@ def test_out_of_order_messages_with_end_message_first_are_handled_in_order(self) message_handler = PubSubEventHandler( subscription=mock_subscription, receiving_service=parent, - message_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, + event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, schema={}, ) @@ -177,7 +177,7 @@ def test_out_of_order_messages_with_end_message_first_are_handled_in_order(self) self.assertEqual(result, "This is the result.") self.assertEqual( - message_handler.handled_messages, + message_handler.handled_events, [ {"kind": "test", "order": 0}, {"kind": "test", "order": 1}, @@ -194,7 +194,7 @@ def test_no_timeout(self): message_handler = PubSubEventHandler( subscription=mock_subscription, receiving_service=parent, - message_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, + event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, schema={}, ) @@ -222,7 +222,7 @@ def test_no_timeout(self): self.assertEqual(result, "This is the result.") self.assertEqual( - message_handler.handled_messages, + message_handler.handled_events, [{"kind": "test", "order": 0}, {"kind": "test", "order": 1}, {"kind": "finish-test", "order": 2}], ) @@ -336,7 +336,7 @@ def test_time_since_missing_message_is_none_if_no_unhandled_missing_messages(sel """ question_uuid, _, mock_subscription = create_mock_topic_and_subscription() message_handler = PubSubEventHandler(subscription=mock_subscription, receiving_service=parent) - self.assertIsNone(message_handler.time_since_missing_message) + self.assertIsNone(message_handler.time_since_missing_event) def test_missing_messages_at_start_can_be_skipped(self): """Test that missing messages at the start of the event stream can be skipped if they aren't received after a @@ -348,9 +348,9 @@ def test_missing_messages_at_start_can_be_skipped(self): message_handler = PubSubEventHandler( subscription=mock_subscription, receiving_service=parent, - message_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, + event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, schema={}, - skip_missing_messages_after=0, + skip_missing_events_after=0, ) # Simulate the first two messages not being received. @@ -384,7 +384,7 @@ def test_missing_messages_at_start_can_be_skipped(self): self.assertEqual(result, "This is the result.") self.assertEqual( - message_handler.handled_messages, + message_handler.handled_events, [ {"kind": "test", "order": 2}, {"kind": "test", "order": 3}, @@ -401,9 +401,9 @@ def test_missing_messages_in_middle_can_skipped(self): message_handler = PubSubEventHandler( subscription=mock_subscription, receiving_service=parent, - message_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, + event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, schema={}, - skip_missing_messages_after=0, + skip_missing_events_after=0, ) child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) @@ -441,7 +441,7 @@ def test_missing_messages_in_middle_can_skipped(self): # Check that all the non-missing messages were handled. self.assertEqual( - message_handler.handled_messages, + message_handler.handled_events, [ {"kind": "test", "order": 0}, {"kind": "test", "order": 1}, @@ -458,9 +458,9 @@ def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): message_handler = PubSubEventHandler( subscription=mock_subscription, receiving_service=parent, - message_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, + event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, schema={}, - skip_missing_messages_after=0, + skip_missing_events_after=0, ) child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) @@ -524,7 +524,7 @@ def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): # Check that all the non-missing messages were handled. self.assertEqual( - message_handler.handled_messages, + message_handler.handled_events, [ {"kind": "test", "order": 0}, {"kind": "test", "order": 1}, @@ -545,9 +545,9 @@ def test_all_messages_missing_apart_from_result(self): message_handler = PubSubEventHandler( subscription=mock_subscription, receiving_service=parent, - message_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, + event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, schema={}, - skip_missing_messages_after=0, + skip_missing_events_after=0, ) child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) @@ -565,11 +565,11 @@ def test_all_messages_missing_apart_from_result(self): message_handler.handle_events() # Check that the result message was handled. - self.assertEqual(message_handler.handled_messages, [{"kind": "finish-test", "order": 1000}]) + self.assertEqual(message_handler.handled_events, [{"kind": "finish-test", "order": 1000}]) class TestPullAndEnqueueAvailableMessages(BaseTestCase): - def test_pull_and_enqueue_available_messages(self): + def test_pull_and_enqueue_available_events(self): """Test that pulling and enqueuing a message works.""" question_uuid, mock_topic, _ = create_mock_topic_and_subscription() @@ -582,12 +582,12 @@ def test_pull_and_enqueue_available_messages(self): message_handler = PubSubEventHandler( subscription=mock_subscription, receiving_service=parent, - message_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, + event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, schema={}, ) message_handler._child_sdk_version = "0.1.3" - message_handler.waiting_messages = {} + message_handler.waiting_events = {} # Enqueue a mock message for a mock subscription to receive. mock_message = {"kind": "test"} @@ -604,9 +604,9 @@ def test_pull_and_enqueue_available_messages(self): ) ] - message_handler._pull_and_enqueue_available_messages(timeout=10) - self.assertEqual(message_handler.waiting_messages, {0: mock_message}) - self.assertEqual(message_handler._earliest_waiting_message_number, 0) + message_handler._pull_and_enqueue_available_events(timeout=10) + self.assertEqual(message_handler.waiting_events, {0: mock_message}) + self.assertEqual(message_handler._earliest_waiting_event_number, 0) def test_timeout_error_raised_if_result_message_not_received_in_time(self): """Test that a timeout error is raised if a result message is not received in time.""" @@ -621,17 +621,17 @@ def test_timeout_error_raised_if_result_message_not_received_in_time(self): message_handler = PubSubEventHandler( subscription=mock_subscription, receiving_service=parent, - message_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, + event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, ) message_handler._child_sdk_version = "0.1.3" - message_handler.waiting_messages = {} + message_handler.waiting_events = {} message_handler._start_time = 0 # Create a mock subscription. SUBSCRIPTIONS[mock_subscription.name] = [] with self.assertRaises(TimeoutError): - message_handler._pull_and_enqueue_available_messages(timeout=1e-6) + message_handler._pull_and_enqueue_available_events(timeout=1e-6) - self.assertEqual(message_handler._earliest_waiting_message_number, math.inf) + self.assertEqual(message_handler._earliest_waiting_event_number, math.inf) From b4ce92c8083f4071ef778061013f58065d4e728b Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 11 Mar 2024 10:05:19 +0000 Subject: [PATCH 038/169] REF: Rename `PubSubEventHandler` to `GoogleCloudPubSubEventHandler` --- octue/cloud/pub_sub/event_handler.py | 2 +- octue/cloud/pub_sub/service.py | 4 +-- tests/cloud/pub_sub/test_message_handler.py | 38 ++++++++++----------- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/octue/cloud/pub_sub/event_handler.py b/octue/cloud/pub_sub/event_handler.py index 7e9da7070..f4f4c7051 100644 --- a/octue/cloud/pub_sub/event_handler.py +++ b/octue/cloud/pub_sub/event_handler.py @@ -19,7 +19,7 @@ PARENT_SDK_VERSION = importlib.metadata.version("octue") -class PubSubEventHandler(AbstractEventHandler): +class GoogleCloudPubSubEventHandler(AbstractEventHandler): """A handler for events received as Google Pub/Sub messages from a pull subscription. :param octue.cloud.pub_sub.subscription.Subscription subscription: the subscription messages are pulled from diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index b4dc1eea8..fa94d9400 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -16,7 +16,7 @@ import octue.exceptions from octue.cloud.events.validation import raise_if_event_is_invalid from octue.cloud.pub_sub import Subscription, Topic -from octue.cloud.pub_sub.event_handler import PubSubEventHandler +from octue.cloud.pub_sub.event_handler import GoogleCloudPubSubEventHandler from octue.cloud.pub_sub.events import extract_event_and_attributes_from_pub_sub from octue.cloud.pub_sub.logging import GooglePubSubHandler from octue.cloud.service_id import ( @@ -391,7 +391,7 @@ def wait_for_answer( service_name = get_sruid_from_pub_sub_resource_name(subscription.name) - self._event_handler = PubSubEventHandler( + self._event_handler = GoogleCloudPubSubEventHandler( subscription=subscription, receiving_service=self, handle_monitor_message=handle_monitor_message, diff --git a/tests/cloud/pub_sub/test_message_handler.py b/tests/cloud/pub_sub/test_message_handler.py index e5ab4f7d4..ab47ad157 100644 --- a/tests/cloud/pub_sub/test_message_handler.py +++ b/tests/cloud/pub_sub/test_message_handler.py @@ -12,7 +12,7 @@ MockTopic, ) from octue.cloud.emulators.child import ServicePatcher -from octue.cloud.pub_sub.event_handler import PubSubEventHandler +from octue.cloud.pub_sub.event_handler import GoogleCloudPubSubEventHandler from octue.resources.service_backends import GCPPubSubBackend from tests import TEST_PROJECT_NAME from tests.base import BaseTestCase @@ -40,7 +40,7 @@ def test_timeout(self): question_uuid, _, mock_subscription = create_mock_topic_and_subscription() with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): - message_handler = PubSubEventHandler( + message_handler = GoogleCloudPubSubEventHandler( subscription=mock_subscription, receiving_service=parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: message}, @@ -55,7 +55,7 @@ def test_in_order_messages_are_handled_in_order(self): question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): - message_handler = PubSubEventHandler( + message_handler = GoogleCloudPubSubEventHandler( subscription=mock_subscription, receiving_service=parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, @@ -87,7 +87,7 @@ def test_out_of_order_messages_are_handled_in_order(self): question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): - message_handler = PubSubEventHandler( + message_handler = GoogleCloudPubSubEventHandler( subscription=mock_subscription, receiving_service=parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, @@ -140,7 +140,7 @@ def test_out_of_order_messages_with_end_message_first_are_handled_in_order(self) question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): - message_handler = PubSubEventHandler( + message_handler = GoogleCloudPubSubEventHandler( subscription=mock_subscription, receiving_service=parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, @@ -191,7 +191,7 @@ def test_no_timeout(self): question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): - message_handler = PubSubEventHandler( + message_handler = GoogleCloudPubSubEventHandler( subscription=mock_subscription, receiving_service=parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, @@ -231,7 +231,7 @@ def test_delivery_acknowledgement(self): question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): - message_handler = PubSubEventHandler(subscription=mock_subscription, receiving_service=parent) + message_handler = GoogleCloudPubSubEventHandler(subscription=mock_subscription, receiving_service=parent) child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) @@ -257,7 +257,7 @@ def test_error_raised_if_heartbeat_not_received_before_checked(self): question_uuid, _, mock_subscription = create_mock_topic_and_subscription() with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): - message_handler = PubSubEventHandler(subscription=mock_subscription, receiving_service=parent) + message_handler = GoogleCloudPubSubEventHandler(subscription=mock_subscription, receiving_service=parent) with self.assertRaises(TimeoutError) as error: message_handler.handle_events(maximum_heartbeat_interval=0) @@ -270,7 +270,7 @@ def test_error_raised_if_heartbeats_stop_being_received(self): question_uuid, _, mock_subscription = create_mock_topic_and_subscription() with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): - message_handler = PubSubEventHandler(subscription=mock_subscription, receiving_service=parent) + message_handler = GoogleCloudPubSubEventHandler(subscription=mock_subscription, receiving_service=parent) message_handler._last_heartbeat = datetime.datetime.now() - datetime.timedelta(seconds=30) @@ -284,7 +284,7 @@ def test_error_not_raised_if_heartbeat_has_been_received_in_maximum_allowed_inte question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): - message_handler = PubSubEventHandler(subscription=mock_subscription, receiving_service=parent) + message_handler = GoogleCloudPubSubEventHandler(subscription=mock_subscription, receiving_service=parent) message_handler._last_heartbeat = datetime.datetime.now() @@ -318,7 +318,7 @@ def test_time_since_last_heartbeat_is_none_if_no_heartbeat_received_yet(self): question_uuid, _, mock_subscription = create_mock_topic_and_subscription() with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): - message_handler = PubSubEventHandler(subscription=mock_subscription, receiving_service=parent) + message_handler = GoogleCloudPubSubEventHandler(subscription=mock_subscription, receiving_service=parent) self.assertIsNone(message_handler._time_since_last_heartbeat) @@ -327,7 +327,7 @@ def test_total_run_time_is_none_if_handle_events_has_not_been_called(self): called. """ question_uuid, _, mock_subscription = create_mock_topic_and_subscription() - message_handler = PubSubEventHandler(subscription=mock_subscription, receiving_service=parent) + message_handler = GoogleCloudPubSubEventHandler(subscription=mock_subscription, receiving_service=parent) self.assertIsNone(message_handler.total_run_time) def test_time_since_missing_message_is_none_if_no_unhandled_missing_messages(self): @@ -335,7 +335,7 @@ def test_time_since_missing_message_is_none_if_no_unhandled_missing_messages(sel missing messages. """ question_uuid, _, mock_subscription = create_mock_topic_and_subscription() - message_handler = PubSubEventHandler(subscription=mock_subscription, receiving_service=parent) + message_handler = GoogleCloudPubSubEventHandler(subscription=mock_subscription, receiving_service=parent) self.assertIsNone(message_handler.time_since_missing_event) def test_missing_messages_at_start_can_be_skipped(self): @@ -345,7 +345,7 @@ def test_missing_messages_at_start_can_be_skipped(self): question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): - message_handler = PubSubEventHandler( + message_handler = GoogleCloudPubSubEventHandler( subscription=mock_subscription, receiving_service=parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, @@ -398,7 +398,7 @@ def test_missing_messages_in_middle_can_skipped(self): question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): - message_handler = PubSubEventHandler( + message_handler = GoogleCloudPubSubEventHandler( subscription=mock_subscription, receiving_service=parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, @@ -455,7 +455,7 @@ def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): - message_handler = PubSubEventHandler( + message_handler = GoogleCloudPubSubEventHandler( subscription=mock_subscription, receiving_service=parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, @@ -542,7 +542,7 @@ def test_all_messages_missing_apart_from_result(self): question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): - message_handler = PubSubEventHandler( + message_handler = GoogleCloudPubSubEventHandler( subscription=mock_subscription, receiving_service=parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, @@ -579,7 +579,7 @@ def test_pull_and_enqueue_available_events(self): topic=mock_topic, ) - message_handler = PubSubEventHandler( + message_handler = GoogleCloudPubSubEventHandler( subscription=mock_subscription, receiving_service=parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, @@ -618,7 +618,7 @@ def test_timeout_error_raised_if_result_message_not_received_in_time(self): topic=mock_topic, ) - message_handler = PubSubEventHandler( + message_handler = GoogleCloudPubSubEventHandler( subscription=mock_subscription, receiving_service=parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, From c61f5d97dc380cad497bac6d5749f6df2a23ab78 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 11 Mar 2024 10:06:53 +0000 Subject: [PATCH 039/169] REF: Rename `GooglePubSubHandler` to `GoogleCloudPubSubHandler` --- octue/cloud/pub_sub/logging.py | 2 +- octue/cloud/pub_sub/service.py | 4 ++-- tests/cloud/pub_sub/test_logging.py | 10 +++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/octue/cloud/pub_sub/logging.py b/octue/cloud/pub_sub/logging.py index 286269cd0..4df17b92e 100644 --- a/octue/cloud/pub_sub/logging.py +++ b/octue/cloud/pub_sub/logging.py @@ -5,7 +5,7 @@ ANSI_ESCAPE_SEQUENCES_PATTERN = r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])" -class GooglePubSubHandler(logging.Handler): +class GoogleCloudPubSubHandler(logging.Handler): """A log handler that publishes log records to a Google Cloud Pub/Sub topic. :param callable message_sender: the `_send_message` method of the service that instantiated this instance diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index fa94d9400..bc33b01e5 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -18,7 +18,7 @@ from octue.cloud.pub_sub import Subscription, Topic from octue.cloud.pub_sub.event_handler import GoogleCloudPubSubEventHandler from octue.cloud.pub_sub.events import extract_event_and_attributes_from_pub_sub -from octue.cloud.pub_sub.logging import GooglePubSubHandler +from octue.cloud.pub_sub.logging import GoogleCloudPubSubHandler from octue.cloud.service_id import ( convert_service_id_to_pub_sub_form, create_sruid, @@ -226,7 +226,7 @@ def answer(self, question, heartbeat_interval=120, timeout=30): heartbeater.start() if forward_logs: - analysis_log_handler = GooglePubSubHandler( + analysis_log_handler = GoogleCloudPubSubHandler( message_sender=self._send_message, topic=topic, question_uuid=question_uuid, diff --git a/tests/cloud/pub_sub/test_logging.py b/tests/cloud/pub_sub/test_logging.py index 4e6e9f491..5b333cb3c 100644 --- a/tests/cloud/pub_sub/test_logging.py +++ b/tests/cloud/pub_sub/test_logging.py @@ -4,7 +4,7 @@ from unittest.mock import patch from octue.cloud.emulators._pub_sub import SUBSCRIPTIONS, MockService, MockSubscription, MockTopic -from octue.cloud.pub_sub.logging import GooglePubSubHandler +from octue.cloud.pub_sub.logging import GoogleCloudPubSubHandler from octue.resources.service_backends import GCPPubSubBackend from tests.base import BaseTestCase @@ -14,9 +14,9 @@ def __repr__(self): return "NonJSONSerialisableInstance" -class TestGooglePubSubHandler(BaseTestCase): +class TestGoogleCloudPubSubHandler(BaseTestCase): def test_emit(self): - """Test the log message is published when `GooglePubSubHandler.emit` is called.""" + """Test the log message is published when `GoogleCloudPubSubHandler.emit` is called.""" topic = MockTopic(name="world", project_name="blah") topic.create() @@ -29,7 +29,7 @@ def test_emit(self): backend = GCPPubSubBackend(project_name="blah") service = MockService(backend=backend) - GooglePubSubHandler( + GoogleCloudPubSubHandler( message_sender=service._send_message, topic=topic, question_uuid=question_uuid, @@ -61,7 +61,7 @@ def test_emit_with_non_json_serialisable_args(self): service = MockService(backend=backend) with patch("octue.cloud.emulators._pub_sub.MockPublisher.publish") as mock_publish: - GooglePubSubHandler( + GoogleCloudPubSubHandler( message_sender=service._send_message, topic=topic, question_uuid="question-uuid", From 945e4665a4fe4161560832d9c1e8cb82d16b4c0c Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 11 Mar 2024 10:50:58 +0000 Subject: [PATCH 040/169] ENH: Allow skipping of non-result events in event handler --- octue/cloud/events/handler.py | 5 +++++ octue/cloud/events/replayer.py | 2 ++ 2 files changed, 7 insertions(+) diff --git a/octue/cloud/events/handler.py b/octue/cloud/events/handler.py index 1e23e0a80..d9e3000bb 100644 --- a/octue/cloud/events/handler.py +++ b/octue/cloud/events/handler.py @@ -39,12 +39,14 @@ def __init__( event_handlers=None, schema=SERVICE_COMMUNICATION_SCHEMA, skip_missing_events_after=10, + only_handle_result=False, ): self.receiving_service = receiving_service self.handle_monitor_message = handle_monitor_message self.record_events = record_events self.service_name = service_name self.schema = schema + self.only_handle_result = only_handle_result self.waiting_events = None self.handled_events = [] @@ -195,6 +197,9 @@ def _handle_event(self, event): if self.record_events: self.handled_events.append(event) + if self.only_handle_result and event["kind"] != "result": + return + handler = self._event_handlers[event["kind"]] return handler(event) diff --git a/octue/cloud/events/replayer.py b/octue/cloud/events/replayer.py index 7d977d4d0..4bcf175fd 100644 --- a/octue/cloud/events/replayer.py +++ b/octue/cloud/events/replayer.py @@ -18,6 +18,7 @@ def __init__( service_name="REMOTE", event_handlers=None, schema=SERVICE_COMMUNICATION_SCHEMA, + only_handle_result=False, ): super().__init__( receiving_service or Service(backend=ServiceBackend(), service_id="local/local:local"), @@ -27,6 +28,7 @@ def __init__( event_handlers=event_handlers, schema=schema, skip_missing_events_after=0, + only_handle_result=only_handle_result, ) def handle_events(self, events): From c44b80c4aa0ad13c04bd18d569b0eb9fd8bf0da5 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 11 Mar 2024 10:58:39 +0000 Subject: [PATCH 041/169] ENH: Send/get sender SRUID in/from event attributes --- octue/cloud/events/handler.py | 12 ++++++------ octue/cloud/events/replayer.py | 2 -- octue/cloud/pub_sub/service.py | 1 + tests/data/events.json | 24 ++++++++++++++++-------- 4 files changed, 23 insertions(+), 16 deletions(-) diff --git a/octue/cloud/events/handler.py b/octue/cloud/events/handler.py index d9e3000bb..20113f715 100644 --- a/octue/cloud/events/handler.py +++ b/octue/cloud/events/handler.py @@ -35,7 +35,6 @@ def __init__( receiving_service, handle_monitor_message=None, record_events=True, - service_name="REMOTE", event_handlers=None, schema=SERVICE_COMMUNICATION_SCHEMA, skip_missing_events_after=10, @@ -44,7 +43,7 @@ def __init__( self.receiving_service = receiving_service self.handle_monitor_message = handle_monitor_message self.record_events = record_events - self.service_name = service_name + self.child_sruid = None self.schema = schema self.only_handle_result = only_handle_result @@ -107,8 +106,9 @@ def _extract_and_enqueue_event(self, event): ): return - # Get the child's Octue SDK version from the first event. + # Get the child's SRUID and Octue SDK version from the first event. if not self._child_sdk_version: + self.child_sruid = attributes.get("sender") # Backwards-compatible with previous event schema versions. self._child_sdk_version = attributes["version"] event_number = attributes["message_number"] @@ -218,7 +218,7 @@ def _handle_heartbeat(self, event): :return None: """ self._last_heartbeat = datetime.now() - logger.info("Heartbeat received from service %r for question %r.", self.service_name, self.question_uuid) + logger.info("Heartbeat received from service %r for question %r.", self.child_sruid, self.question_uuid) def _handle_monitor_message(self, event): """Send a monitor message to the handler if one has been provided. @@ -243,7 +243,7 @@ def _handle_log_message(self, event): # Add information about the immediate child sending the event and colour it with the first colour in the # colour palette. immediate_child_analysis_section = colourise( - f"[{self.service_name} | analysis-{self.question_uuid}]", + f"[{self.child_sruid} | analysis-{self.question_uuid}]", text_colour=self._log_message_colours[0], ) @@ -270,7 +270,7 @@ def _handle_exception(self, event): exception_message = "\n\n".join( ( event["exception_message"], - f"The following traceback was captured from the remote service {self.service_name!r}:", + f"The following traceback was captured from the remote service {self.child_sruid!r}:", "".join(event["exception_traceback"]), ) ) diff --git a/octue/cloud/events/replayer.py b/octue/cloud/events/replayer.py index 4bcf175fd..d688020fb 100644 --- a/octue/cloud/events/replayer.py +++ b/octue/cloud/events/replayer.py @@ -15,7 +15,6 @@ def __init__( receiving_service=None, handle_monitor_message=None, record_events=True, - service_name="REMOTE", event_handlers=None, schema=SERVICE_COMMUNICATION_SCHEMA, only_handle_result=False, @@ -24,7 +23,6 @@ def __init__( receiving_service or Service(backend=ServiceBackend(), service_id="local/local:local"), handle_monitor_message=handle_monitor_message, record_events=record_events, - service_name=service_name, event_handlers=event_handlers, schema=schema, skip_missing_events_after=0, diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index bc33b01e5..3db5b7dd6 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -446,6 +446,7 @@ def _send_message(self, message, topic, attributes=None, timeout=30): with send_message_lock: attributes["version"] = self._local_sdk_version attributes["message_number"] = topic.messages_published + attributes["sender"] = self.id converted_attributes = {} for key, value in attributes.items(): diff --git a/tests/data/events.json b/tests/data/events.json index 81396a7ac..d1e9e1b2a 100644 --- a/tests/data/events.json +++ b/tests/data/events.json @@ -8,7 +8,8 @@ "message_number": "0", "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", "sender_type": "CHILD", - "version": "0.51.0" + "version": "0.51.0", + "sender": "octue/test-service:1.0.0" } }, { @@ -41,7 +42,8 @@ "message_number": "1", "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", "sender_type": "CHILD", - "version": "0.51.0" + "version": "0.51.0", + "sender": "octue/test-service:1.0.0" } }, { @@ -74,7 +76,8 @@ "message_number": "2", "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", "sender_type": "CHILD", - "version": "0.51.0" + "version": "0.51.0", + "sender": "octue/test-service:1.0.0" } }, { @@ -107,7 +110,8 @@ "message_number": "3", "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", "sender_type": "CHILD", - "version": "0.51.0" + "version": "0.51.0", + "sender": "octue/test-service:1.0.0" } }, { @@ -140,7 +144,8 @@ "message_number": "4", "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", "sender_type": "CHILD", - "version": "0.51.0" + "version": "0.51.0", + "sender": "octue/test-service:1.0.0" } }, { @@ -173,7 +178,8 @@ "message_number": "5", "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", "sender_type": "CHILD", - "version": "0.51.0" + "version": "0.51.0", + "sender": "octue/test-service:1.0.0" } }, { @@ -201,7 +207,8 @@ "message_number": "6", "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", "sender_type": "CHILD", - "version": "0.51.0" + "version": "0.51.0", + "sender": "octue/test-service:1.0.0" } }, { @@ -213,7 +220,8 @@ "message_number": "7", "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", "sender_type": "CHILD", - "version": "0.51.0" + "version": "0.51.0", + "sender": "octue/test-service:1.0.0" } } ] From bd00029a70ea2498ebf707191c1b54e43ff904fb Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 11 Mar 2024 11:05:54 +0000 Subject: [PATCH 042/169] TST: Rename test module --- ...ssage_handler.py => test_event_handler.py} | 108 +++++++++--------- 1 file changed, 54 insertions(+), 54 deletions(-) rename tests/cloud/pub_sub/{test_message_handler.py => test_event_handler.py} (86%) diff --git a/tests/cloud/pub_sub/test_message_handler.py b/tests/cloud/pub_sub/test_event_handler.py similarity index 86% rename from tests/cloud/pub_sub/test_message_handler.py rename to tests/cloud/pub_sub/test_event_handler.py index ab47ad157..492c49b3c 100644 --- a/tests/cloud/pub_sub/test_message_handler.py +++ b/tests/cloud/pub_sub/test_event_handler.py @@ -40,7 +40,7 @@ def test_timeout(self): question_uuid, _, mock_subscription = create_mock_topic_and_subscription() with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): - message_handler = GoogleCloudPubSubEventHandler( + event_handler = GoogleCloudPubSubEventHandler( subscription=mock_subscription, receiving_service=parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: message}, @@ -48,14 +48,14 @@ def test_timeout(self): ) with self.assertRaises(TimeoutError): - message_handler.handle_events(timeout=0) + event_handler.handle_events(timeout=0) def test_in_order_messages_are_handled_in_order(self): """Test that messages received in order are handled in order.""" question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): - message_handler = GoogleCloudPubSubEventHandler( + event_handler = GoogleCloudPubSubEventHandler( subscription=mock_subscription, receiving_service=parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, @@ -74,11 +74,11 @@ def test_in_order_messages_are_handled_in_order(self): for message in messages: child._send_message(message=message["event"], attributes=message["attributes"], topic=mock_topic) - result = message_handler.handle_events() + result = event_handler.handle_events() self.assertEqual(result, "This is the result.") self.assertEqual( - message_handler.handled_events, + event_handler.handled_events, [{"kind": "test"}, {"kind": "test"}, {"kind": "test"}, {"kind": "finish-test"}], ) @@ -87,7 +87,7 @@ def test_out_of_order_messages_are_handled_in_order(self): question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): - message_handler = GoogleCloudPubSubEventHandler( + event_handler = GoogleCloudPubSubEventHandler( subscription=mock_subscription, receiving_service=parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, @@ -119,12 +119,12 @@ def test_out_of_order_messages_are_handled_in_order(self): mock_topic.messages_published = message["event"]["order"] child._send_message(message=message["event"], attributes=message["attributes"], topic=mock_topic) - result = message_handler.handle_events() + result = event_handler.handle_events() self.assertEqual(result, "This is the result.") self.assertEqual( - message_handler.handled_events, + event_handler.handled_events, [ {"kind": "test", "order": 0}, {"kind": "test", "order": 1}, @@ -140,7 +140,7 @@ def test_out_of_order_messages_with_end_message_first_are_handled_in_order(self) question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): - message_handler = GoogleCloudPubSubEventHandler( + event_handler = GoogleCloudPubSubEventHandler( subscription=mock_subscription, receiving_service=parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, @@ -172,12 +172,12 @@ def test_out_of_order_messages_with_end_message_first_are_handled_in_order(self) mock_topic.messages_published = message["event"]["order"] child._send_message(message=message["event"], attributes=message["attributes"], topic=mock_topic) - result = message_handler.handle_events() + result = event_handler.handle_events() self.assertEqual(result, "This is the result.") self.assertEqual( - message_handler.handled_events, + event_handler.handled_events, [ {"kind": "test", "order": 0}, {"kind": "test", "order": 1}, @@ -191,7 +191,7 @@ def test_no_timeout(self): question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): - message_handler = GoogleCloudPubSubEventHandler( + event_handler = GoogleCloudPubSubEventHandler( subscription=mock_subscription, receiving_service=parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, @@ -218,11 +218,11 @@ def test_no_timeout(self): for message in messages: child._send_message(message=message["event"], attributes=message["attributes"], topic=mock_topic) - result = message_handler.handle_events(timeout=None) + result = event_handler.handle_events(timeout=None) self.assertEqual(result, "This is the result.") self.assertEqual( - message_handler.handled_events, + event_handler.handled_events, [{"kind": "test", "order": 0}, {"kind": "test", "order": 1}, {"kind": "finish-test", "order": 2}], ) @@ -231,7 +231,7 @@ def test_delivery_acknowledgement(self): question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): - message_handler = GoogleCloudPubSubEventHandler(subscription=mock_subscription, receiving_service=parent) + event_handler = GoogleCloudPubSubEventHandler(subscription=mock_subscription, receiving_service=parent) child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) @@ -249,7 +249,7 @@ def test_delivery_acknowledgement(self): for message in messages: child._send_message(message=message["event"], attributes=message["attributes"], topic=mock_topic) - result = message_handler.handle_events() + result = event_handler.handle_events() self.assertEqual(result, {"output_values": None, "output_manifest": None}) def test_error_raised_if_heartbeat_not_received_before_checked(self): @@ -257,10 +257,10 @@ def test_error_raised_if_heartbeat_not_received_before_checked(self): question_uuid, _, mock_subscription = create_mock_topic_and_subscription() with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): - message_handler = GoogleCloudPubSubEventHandler(subscription=mock_subscription, receiving_service=parent) + event_handler = GoogleCloudPubSubEventHandler(subscription=mock_subscription, receiving_service=parent) with self.assertRaises(TimeoutError) as error: - message_handler.handle_events(maximum_heartbeat_interval=0) + event_handler.handle_events(maximum_heartbeat_interval=0) # Check that the timeout is due to a heartbeat not being received. self.assertIn("heartbeat", error.exception.args[0]) @@ -270,12 +270,12 @@ def test_error_raised_if_heartbeats_stop_being_received(self): question_uuid, _, mock_subscription = create_mock_topic_and_subscription() with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): - message_handler = GoogleCloudPubSubEventHandler(subscription=mock_subscription, receiving_service=parent) + event_handler = GoogleCloudPubSubEventHandler(subscription=mock_subscription, receiving_service=parent) - message_handler._last_heartbeat = datetime.datetime.now() - datetime.timedelta(seconds=30) + event_handler._last_heartbeat = datetime.datetime.now() - datetime.timedelta(seconds=30) with self.assertRaises(TimeoutError) as error: - message_handler.handle_events(maximum_heartbeat_interval=0) + event_handler.handle_events(maximum_heartbeat_interval=0) self.assertIn("heartbeat", error.exception.args[0]) @@ -284,9 +284,9 @@ def test_error_not_raised_if_heartbeat_has_been_received_in_maximum_allowed_inte question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): - message_handler = GoogleCloudPubSubEventHandler(subscription=mock_subscription, receiving_service=parent) + event_handler = GoogleCloudPubSubEventHandler(subscription=mock_subscription, receiving_service=parent) - message_handler._last_heartbeat = datetime.datetime.now() + event_handler._last_heartbeat = datetime.datetime.now() child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) @@ -311,32 +311,32 @@ def test_error_not_raised_if_heartbeat_has_been_received_in_maximum_allowed_inte "octue.cloud.pub_sub.event_handler.PubSubEventHandler._time_since_last_heartbeat", datetime.timedelta(seconds=0), ): - message_handler.handle_events(maximum_heartbeat_interval=0) + event_handler.handle_events(maximum_heartbeat_interval=0) def test_time_since_last_heartbeat_is_none_if_no_heartbeat_received_yet(self): """Test that the time since the last heartbeat is `None` if no heartbeat has been received yet.""" question_uuid, _, mock_subscription = create_mock_topic_and_subscription() with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): - message_handler = GoogleCloudPubSubEventHandler(subscription=mock_subscription, receiving_service=parent) + event_handler = GoogleCloudPubSubEventHandler(subscription=mock_subscription, receiving_service=parent) - self.assertIsNone(message_handler._time_since_last_heartbeat) + self.assertIsNone(event_handler._time_since_last_heartbeat) def test_total_run_time_is_none_if_handle_events_has_not_been_called(self): """Test that the total run time for the message handler is `None` if the `handle_events` method has not been called. """ question_uuid, _, mock_subscription = create_mock_topic_and_subscription() - message_handler = GoogleCloudPubSubEventHandler(subscription=mock_subscription, receiving_service=parent) - self.assertIsNone(message_handler.total_run_time) + event_handler = GoogleCloudPubSubEventHandler(subscription=mock_subscription, receiving_service=parent) + self.assertIsNone(event_handler.total_run_time) def test_time_since_missing_message_is_none_if_no_unhandled_missing_messages(self): """Test that the `PubSubEventHandler.time_since_missing_message` property is `None` if there are no unhandled missing messages. """ question_uuid, _, mock_subscription = create_mock_topic_and_subscription() - message_handler = GoogleCloudPubSubEventHandler(subscription=mock_subscription, receiving_service=parent) - self.assertIsNone(message_handler.time_since_missing_event) + event_handler = GoogleCloudPubSubEventHandler(subscription=mock_subscription, receiving_service=parent) + self.assertIsNone(event_handler.time_since_missing_event) def test_missing_messages_at_start_can_be_skipped(self): """Test that missing messages at the start of the event stream can be skipped if they aren't received after a @@ -345,7 +345,7 @@ def test_missing_messages_at_start_can_be_skipped(self): question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): - message_handler = GoogleCloudPubSubEventHandler( + event_handler = GoogleCloudPubSubEventHandler( subscription=mock_subscription, receiving_service=parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, @@ -380,11 +380,11 @@ def test_missing_messages_at_start_can_be_skipped(self): for message in messages: child._send_message(message=message["event"], attributes=message["attributes"], topic=mock_topic) - result = message_handler.handle_events() + result = event_handler.handle_events() self.assertEqual(result, "This is the result.") self.assertEqual( - message_handler.handled_events, + event_handler.handled_events, [ {"kind": "test", "order": 2}, {"kind": "test", "order": 3}, @@ -398,7 +398,7 @@ def test_missing_messages_in_middle_can_skipped(self): question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): - message_handler = GoogleCloudPubSubEventHandler( + event_handler = GoogleCloudPubSubEventHandler( subscription=mock_subscription, receiving_service=parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, @@ -437,11 +437,11 @@ def test_missing_messages_in_middle_can_skipped(self): topic=mock_topic, ) - message_handler.handle_events() + event_handler.handle_events() # Check that all the non-missing messages were handled. self.assertEqual( - message_handler.handled_events, + event_handler.handled_events, [ {"kind": "test", "order": 0}, {"kind": "test", "order": 1}, @@ -455,7 +455,7 @@ def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): - message_handler = GoogleCloudPubSubEventHandler( + event_handler = GoogleCloudPubSubEventHandler( subscription=mock_subscription, receiving_service=parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, @@ -520,11 +520,11 @@ def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): for message in messages: child._send_message(message=message["event"], attributes=message["attributes"], topic=mock_topic) - message_handler.handle_events() + event_handler.handle_events() # Check that all the non-missing messages were handled. self.assertEqual( - message_handler.handled_events, + event_handler.handled_events, [ {"kind": "test", "order": 0}, {"kind": "test", "order": 1}, @@ -542,7 +542,7 @@ def test_all_messages_missing_apart_from_result(self): question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): - message_handler = GoogleCloudPubSubEventHandler( + event_handler = GoogleCloudPubSubEventHandler( subscription=mock_subscription, receiving_service=parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, @@ -562,10 +562,10 @@ def test_all_messages_missing_apart_from_result(self): topic=mock_topic, ) - message_handler.handle_events() + event_handler.handle_events() # Check that the result message was handled. - self.assertEqual(message_handler.handled_events, [{"kind": "finish-test", "order": 1000}]) + self.assertEqual(event_handler.handled_events, [{"kind": "finish-test", "order": 1000}]) class TestPullAndEnqueueAvailableMessages(BaseTestCase): @@ -579,15 +579,15 @@ def test_pull_and_enqueue_available_events(self): topic=mock_topic, ) - message_handler = GoogleCloudPubSubEventHandler( + event_handler = GoogleCloudPubSubEventHandler( subscription=mock_subscription, receiving_service=parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, schema={}, ) - message_handler._child_sdk_version = "0.1.3" - message_handler.waiting_events = {} + event_handler._child_sdk_version = "0.1.3" + event_handler.waiting_events = {} # Enqueue a mock message for a mock subscription to receive. mock_message = {"kind": "test"} @@ -604,9 +604,9 @@ def test_pull_and_enqueue_available_events(self): ) ] - message_handler._pull_and_enqueue_available_events(timeout=10) - self.assertEqual(message_handler.waiting_events, {0: mock_message}) - self.assertEqual(message_handler._earliest_waiting_event_number, 0) + event_handler._pull_and_enqueue_available_events(timeout=10) + self.assertEqual(event_handler.waiting_events, {0: mock_message}) + self.assertEqual(event_handler._earliest_waiting_event_number, 0) def test_timeout_error_raised_if_result_message_not_received_in_time(self): """Test that a timeout error is raised if a result message is not received in time.""" @@ -618,20 +618,20 @@ def test_timeout_error_raised_if_result_message_not_received_in_time(self): topic=mock_topic, ) - message_handler = GoogleCloudPubSubEventHandler( + event_handler = GoogleCloudPubSubEventHandler( subscription=mock_subscription, receiving_service=parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, ) - message_handler._child_sdk_version = "0.1.3" - message_handler.waiting_events = {} - message_handler._start_time = 0 + event_handler._child_sdk_version = "0.1.3" + event_handler.waiting_events = {} + event_handler._start_time = 0 # Create a mock subscription. SUBSCRIPTIONS[mock_subscription.name] = [] with self.assertRaises(TimeoutError): - message_handler._pull_and_enqueue_available_events(timeout=1e-6) + event_handler._pull_and_enqueue_available_events(timeout=1e-6) - self.assertEqual(message_handler._earliest_waiting_event_number, math.inf) + self.assertEqual(event_handler._earliest_waiting_event_number, math.inf) From 30f59d766db2399b2dc7f31755bb28ae4eef0c98 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 11 Mar 2024 11:10:35 +0000 Subject: [PATCH 043/169] REF: Factor out getting question UUID into `AbstractEventHandler` --- octue/cloud/events/handler.py | 3 ++- octue/cloud/events/replayer.py | 1 - octue/cloud/pub_sub/event_handler.py | 4 ---- tests/cloud/pub_sub/test_event_handler.py | 8 ++++---- 4 files changed, 6 insertions(+), 10 deletions(-) diff --git a/octue/cloud/events/handler.py b/octue/cloud/events/handler.py index 20113f715..be2f423ec 100644 --- a/octue/cloud/events/handler.py +++ b/octue/cloud/events/handler.py @@ -93,7 +93,6 @@ def _extract_and_enqueue_event(self, event): :param dict event: :return None: """ - logger.debug("%r received an event related to question %r.", self.receiving_service, self.question_uuid) event, attributes = self._extract_event_and_attributes(event) if not is_event_valid( @@ -108,9 +107,11 @@ def _extract_and_enqueue_event(self, event): # Get the child's SRUID and Octue SDK version from the first event. if not self._child_sdk_version: + self.question_uuid = attributes.get("question_uuid") self.child_sruid = attributes.get("sender") # Backwards-compatible with previous event schema versions. self._child_sdk_version = attributes["version"] + logger.debug("%r received an event related to question %r.", self.receiving_service, self.question_uuid) event_number = attributes["message_number"] if event_number in self.waiting_events: diff --git a/octue/cloud/events/replayer.py b/octue/cloud/events/replayer.py index d688020fb..1c4e8c692 100644 --- a/octue/cloud/events/replayer.py +++ b/octue/cloud/events/replayer.py @@ -30,7 +30,6 @@ def __init__( ) def handle_events(self, events): - self.question_uuid = events[0]["attributes"]["question_uuid"] self.waiting_events = {} self._previous_event_number = -1 diff --git a/octue/cloud/pub_sub/event_handler.py b/octue/cloud/pub_sub/event_handler.py index f4f4c7051..60415a85a 100644 --- a/octue/cloud/pub_sub/event_handler.py +++ b/octue/cloud/pub_sub/event_handler.py @@ -26,7 +26,6 @@ class GoogleCloudPubSubEventHandler(AbstractEventHandler): :param octue.cloud.pub_sub.service.Service receiving_service: the service that's receiving the events :param callable|None handle_monitor_message: a function to handle monitor messages (e.g. send them to an endpoint for plotting or displaying) - this function should take a single JSON-compatible python primitive :param bool record_events: if `True`, record received events in the `received_events` attribute - :param str service_name: an arbitrary name to refer to the service subscribed to by (used for labelling its remote log messages) :param dict|None event_handlers: a mapping of event type names to callables that handle each type of event. The handlers should not mutate the events. :param dict|str schema: the JSON schema (or URI of one) to validate events against :param int|float skip_missing_events_after: the number of seconds after which to skip any events if they haven't arrived but subsequent events have @@ -39,7 +38,6 @@ def __init__( receiving_service, handle_monitor_message=None, record_events=True, - service_name="REMOTE", event_handlers=None, schema=SERVICE_COMMUNICATION_SCHEMA, skip_missing_events_after=10, @@ -50,13 +48,11 @@ def __init__( receiving_service, handle_monitor_message=handle_monitor_message, record_events=record_events, - service_name=service_name, event_handlers=event_handlers, schema=schema, skip_missing_events_after=skip_missing_events_after, ) - self.question_uuid = self.subscription.path.split(".")[-1] self.waiting_events = None self._subscriber = SubscriberClient() self._heartbeat_checker = None diff --git a/tests/cloud/pub_sub/test_event_handler.py b/tests/cloud/pub_sub/test_event_handler.py index 492c49b3c..8047a225b 100644 --- a/tests/cloud/pub_sub/test_event_handler.py +++ b/tests/cloud/pub_sub/test_event_handler.py @@ -308,7 +308,7 @@ def test_error_not_raised_if_heartbeat_has_been_received_in_maximum_allowed_inte child._send_message(message=message["event"], attributes=message["attributes"], topic=mock_topic) with patch( - "octue.cloud.pub_sub.event_handler.PubSubEventHandler._time_since_last_heartbeat", + "octue.cloud.pub_sub.event_handler.GoogleCloudPubSubEventHandler._time_since_last_heartbeat", datetime.timedelta(seconds=0), ): event_handler.handle_events(maximum_heartbeat_interval=0) @@ -331,9 +331,7 @@ def test_total_run_time_is_none_if_handle_events_has_not_been_called(self): self.assertIsNone(event_handler.total_run_time) def test_time_since_missing_message_is_none_if_no_unhandled_missing_messages(self): - """Test that the `PubSubEventHandler.time_since_missing_message` property is `None` if there are no unhandled - missing messages. - """ + """Test that the `time_since_missing_message` property is `None` if there are no unhandled missing messages.""" question_uuid, _, mock_subscription = create_mock_topic_and_subscription() event_handler = GoogleCloudPubSubEventHandler(subscription=mock_subscription, receiving_service=parent) self.assertIsNone(event_handler.time_since_missing_event) @@ -586,6 +584,8 @@ def test_pull_and_enqueue_available_events(self): schema={}, ) + event_handler.question_uuid = question_uuid + event_handler.child_sruid = "my-org/my-service:1.0.0" event_handler._child_sdk_version = "0.1.3" event_handler.waiting_events = {} From 68f7f3c6ca355bd0a9fd29b51930300772381bb2 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 11 Mar 2024 11:23:51 +0000 Subject: [PATCH 044/169] FIX: Update usage of event handler in `Service` --- octue/cloud/pub_sub/service.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index 3db5b7dd6..7bf7f571f 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -23,7 +23,6 @@ convert_service_id_to_pub_sub_form, create_sruid, get_default_sruid, - get_sruid_from_pub_sub_resource_name, raise_if_revision_not_registered, split_service_id, validate_sruid, @@ -389,13 +388,10 @@ def wait_for_answer( f"check its BigQuery table {subscription.bigquery_table_id!r}." ) - service_name = get_sruid_from_pub_sub_resource_name(subscription.name) - self._event_handler = GoogleCloudPubSubEventHandler( subscription=subscription, receiving_service=self, handle_monitor_message=handle_monitor_message, - service_name=service_name, record_events=record_messages, ) From 30bec30b62fbad1dac49f2b32e391bc63dfdd76f Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 11 Mar 2024 11:24:11 +0000 Subject: [PATCH 045/169] FIX: Extract new `sender` attribute from Pub/Sub messages --- octue/cloud/emulators/_pub_sub.py | 1 + octue/cloud/pub_sub/events.py | 1 + 2 files changed, 2 insertions(+) diff --git a/octue/cloud/emulators/_pub_sub.py b/octue/cloud/emulators/_pub_sub.py index 7b43f4235..000006cc4 100644 --- a/octue/cloud/emulators/_pub_sub.py +++ b/octue/cloud/emulators/_pub_sub.py @@ -376,6 +376,7 @@ def ask( "version": parent_sdk_version, "save_diagnostics": save_diagnostics, "message_number": 0, + "sender": service_id, }, ) ) diff --git a/octue/cloud/pub_sub/events.py b/octue/cloud/pub_sub/events.py index d61af3876..ebc8f8926 100644 --- a/octue/cloud/pub_sub/events.py +++ b/octue/cloud/pub_sub/events.py @@ -20,6 +20,7 @@ def extract_event_and_attributes_from_pub_sub(message): "question_uuid": attributes["question_uuid"], "message_number": int(attributes["message_number"]), "version": attributes["version"], + "sender": attributes.get("sender"), # Backwards-compatible with previous event schema versions. } if "forward_logs" in attributes: From 3a5970cc6245ae1759678b1e526e84ca72b0eacc Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 11 Mar 2024 14:24:37 +0000 Subject: [PATCH 046/169] ENH: Return download path from `Dataset.download` --- octue/resources/dataset.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/octue/resources/dataset.py b/octue/resources/dataset.py index a2714ac94..ab0b55769 100644 --- a/octue/resources/dataset.py +++ b/octue/resources/dataset.py @@ -332,19 +332,17 @@ def get_file_by_label(self, label): return self.files.one(labels__contains=label) def download(self, local_directory=None): - """Download all files in the dataset into the given local directory. If no path to a local directory is given, - the files will be downloaded to temporary locations. + """Download all files in the dataset. - :param str|None local_directory: - :return None: + :param str|None local_directory: the path to a local directory to download the dataset into; if not provided, the files will be downloaded to a temporary directory + :return str: the absolute path to the local directory """ if not self.exists_in_cloud: raise CloudLocationNotSpecified( f"You can only download files from a cloud dataset. This dataset's path is {self.path!r}." ) - local_directory = local_directory or tempfile.TemporaryDirectory().name - + local_directory = os.path.abspath(local_directory or tempfile.TemporaryDirectory().name) datafiles_and_paths = [] for file in self.files: @@ -354,7 +352,7 @@ def download(self, local_directory=None): path_relative_to_dataset = self._datafile_path_relative_to_self(file, path_type="cloud_path") - local_path = os.path.abspath(os.path.join(local_directory, *path_relative_to_dataset.split("/"))) + local_path = os.path.join(local_directory, *path_relative_to_dataset.split("/")) datafiles_and_paths.append({"datafile": file, "local_path": local_path}) def download_datafile(datafile_and_path): @@ -371,6 +369,7 @@ def download_datafile(datafile_and_path): logger.debug("Downloaded datafile to %r.", path) logger.info("Downloaded %r to %r.", self, local_directory) + return local_directory def to_primitive(self, include_files=True): """Convert the dataset to a dictionary of primitives, converting its files into their paths for a lightweight From 6d98a2937ed505f6d02142fc8aa4b4292af8886d Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 11 Mar 2024 14:25:38 +0000 Subject: [PATCH 047/169] FEA: Add `Manifest.download` method --- octue/resources/manifest.py | 22 +++++++++++++++ tests/resources/test_manifest.py | 48 ++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/octue/resources/manifest.py b/octue/resources/manifest.py index 2fc246315..e38f97623 100644 --- a/octue/resources/manifest.py +++ b/octue/resources/manifest.py @@ -66,6 +66,28 @@ def all_datasets_are_in_cloud(self): """ return all(dataset.all_files_are_in_cloud for dataset in self.datasets.values()) + def download(self, paths=None, download_all=True): + """Download all datasets in the manifest. + + :param dict|None paths: a mapping of dataset name to download directory path; if not provided, datasets are downloaded to temporary directories + :param bool download_all: if `False` and `paths` is provided, only download the datasets specified in `paths` + :return dict(str, str): the downloaded datasets mapped to the absolute paths of the directories they were downloaded into + """ + if paths is None: + download_all = True + paths = {} + + for name, dataset in self.datasets.items(): + download_path = paths.get(name) + + if not download_path and not download_all: + logger.info("%r dataset download skipped as its download path wasn't specified.", name) + continue + + paths[name] = dataset.download(local_directory=download_path) + + return paths + def update_dataset_paths(self, path_generator): """Update the path of each dataset according to the given path generator function. This method is thread-safe. diff --git a/tests/resources/test_manifest.py b/tests/resources/test_manifest.py index 49717cdda..c59a2e823 100644 --- a/tests/resources/test_manifest.py +++ b/tests/resources/test_manifest.py @@ -308,3 +308,51 @@ def test_use_signed_urls_for_datasets_is_idempotent(self): self.assertTrue(manifest.datasets["my_dataset"].path.startswith("http")) self.assertIn(".signed_metadata_files", manifest.datasets["my_dataset"].path) + + def test_download_without_paths(self): + """Test downloading a manifest without specifying the `paths` argument.""" + manifest = Manifest(datasets={"my_dataset": self.create_nested_cloud_dataset()}) + paths = manifest.download() + self.assertEqual(os.listdir(paths["my_dataset"]), ["file_1.txt", "file_0.txt", "sub-directory"]) + + def test_download_with_paths(self): + """Test downloading a manifest while specifying the `paths` argument.""" + manifest = Manifest( + datasets={ + "dataset1": self.create_nested_cloud_dataset(), + "dataset2": self.create_nested_cloud_dataset(), + }, + ) + + with tempfile.TemporaryDirectory() as temporary_directory: + paths = manifest.download(paths={"dataset1": temporary_directory}) + + # Check that dataset1 has been downloaded to the given directory. + self.assertEqual(paths["dataset1"], temporary_directory) + self.assertEqual(os.listdir(paths["dataset1"]), ["file_1.txt", "file_0.txt", "sub-directory"]) + + # Check that dataset2 has been downloaded to a temporary directory. + self.assertNotEqual(paths["dataset2"], temporary_directory) + self.assertEqual(os.listdir(paths["dataset2"]), ["file_1.txt", "file_0.txt", "sub-directory"]) + + def test_download_some_datasets_with_paths(self): + """Test downloading a manifest while specifying the `paths` argument and setting `download_all=False`.""" + manifest = Manifest( + datasets={ + "dataset1": self.create_nested_cloud_dataset(), + "dataset2": self.create_nested_cloud_dataset(), + }, + ) + + with tempfile.TemporaryDirectory() as temporary_directory: + with self.assertLogs() as logging_context: + paths = manifest.download(paths={"dataset1": temporary_directory}, download_all=False) + + # Check that dataset1 has been downloaded to the given directory and dataset2 hasn't been downloaded. + self.assertEqual(paths, {"dataset1": temporary_directory}) + self.assertEqual(os.listdir(paths["dataset1"]), ["file_1.txt", "file_0.txt", "sub-directory"]) + + self.assertEqual( + logging_context.records[1].message, + "'dataset2' dataset download skipped as its download path wasn't specified.", + ) From 0b0266b7ed271128eccf854a3b4674de8a17be8f Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 11 Mar 2024 16:51:38 +0000 Subject: [PATCH 048/169] ENH: Use version `0.8.3` of the service communication event schema --- octue/cloud/events/validation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octue/cloud/events/validation.py b/octue/cloud/events/validation.py index 98137651e..553018401 100644 --- a/octue/cloud/events/validation.py +++ b/octue/cloud/events/validation.py @@ -8,7 +8,7 @@ logger = logging.getLogger(__name__) -SERVICE_COMMUNICATION_SCHEMA = {"$ref": "https://jsonschema.registry.octue.com/octue/service-communication/0.8.2.json"} +SERVICE_COMMUNICATION_SCHEMA = {"$ref": "https://jsonschema.registry.octue.com/octue/service-communication/0.8.3.json"} SERVICE_COMMUNICATION_SCHEMA_INFO_URL = "https://strands.octue.com/octue/service-communication" SERVICE_COMMUNICATION_SCHEMA_VERSION = os.path.splitext(SERVICE_COMMUNICATION_SCHEMA["$ref"])[0].split("/")[-1] From e1e77b6d6705122696e39633c691b987786d8a25 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 11 Mar 2024 16:53:15 +0000 Subject: [PATCH 049/169] FIX: Set `sender` to "REMOTE" when not specified --- octue/cloud/events/handler.py | 4 +++- octue/cloud/pub_sub/events.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/octue/cloud/events/handler.py b/octue/cloud/events/handler.py index be2f423ec..918635cbc 100644 --- a/octue/cloud/events/handler.py +++ b/octue/cloud/events/handler.py @@ -108,9 +108,11 @@ def _extract_and_enqueue_event(self, event): # Get the child's SRUID and Octue SDK version from the first event. if not self._child_sdk_version: self.question_uuid = attributes.get("question_uuid") - self.child_sruid = attributes.get("sender") # Backwards-compatible with previous event schema versions. self._child_sdk_version = attributes["version"] + # Backwards-compatible with previous event schema versions. + self.child_sruid = attributes.get("sender", "REMOTE") + logger.debug("%r received an event related to question %r.", self.receiving_service, self.question_uuid) event_number = attributes["message_number"] diff --git a/octue/cloud/pub_sub/events.py b/octue/cloud/pub_sub/events.py index ebc8f8926..863b12033 100644 --- a/octue/cloud/pub_sub/events.py +++ b/octue/cloud/pub_sub/events.py @@ -20,7 +20,7 @@ def extract_event_and_attributes_from_pub_sub(message): "question_uuid": attributes["question_uuid"], "message_number": int(attributes["message_number"]), "version": attributes["version"], - "sender": attributes.get("sender"), # Backwards-compatible with previous event schema versions. + "sender": attributes.get("sender", "REMOTE"), # Backwards-compatible with previous event schema versions. } if "forward_logs" in attributes: From 958fbef248cd6b065301dd7d2986fd28da7c8318 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 11 Mar 2024 16:53:45 +0000 Subject: [PATCH 050/169] ENH: Include question UUID in delivery acknowledgement log message --- octue/cloud/pub_sub/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index 7bf7f571f..c1266e262 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -529,7 +529,7 @@ def _send_delivery_acknowledgment(self, topic, question_uuid, timeout=30): attributes={"question_uuid": question_uuid, "sender_type": CHILD_SENDER_TYPE}, ) - logger.info("%r acknowledged receipt of question.", self) + logger.info("%r acknowledged receipt of question %r.", self, question_uuid) def _send_heartbeat(self, topic, question_uuid, timeout=30): """Send a heartbeat to the parent, indicating that the service is alive. From 13b9041969d0f3d190fe53737456a05be7dba66c Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 14 Mar 2024 11:36:16 +0000 Subject: [PATCH 051/169] CHO: Update licence year to 2024 --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 966e776cf..2ebc4c551 100644 --- a/LICENSE +++ b/LICENSE @@ -3,7 +3,7 @@ octue-sdk-python Application SDK for python-based apps on the Octue platform MIT License -Copyright (c) 2017-2022 Octue Ltd +Copyright (c) 2017-2024 Octue Ltd Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal From 5bb5fea2c9117b6ae5a9395d215a030e7e06e3a6 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 18 Mar 2024 17:10:23 +0000 Subject: [PATCH 052/169] OPS: Add cloud function and update bigquery table in terraform config --- terraform/bigquery.tf | 42 ++++++++++++++++++++++++++++++------------ terraform/functions.tf | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 12 deletions(-) create mode 100644 terraform/functions.tf diff --git a/terraform/bigquery.tf b/terraform/bigquery.tf index 448594fce..d48391221 100644 --- a/terraform/bigquery.tf +++ b/terraform/bigquery.tf @@ -1,6 +1,6 @@ resource "google_bigquery_dataset" "test_dataset" { dataset_id = "octue_sdk_python_test_dataset" - description = "A dataset for testing BigQuery subscriptions for the Octue SDK." + description = "A dataset for testing storing events for the Octue SDK." location = "EU" labels = { @@ -19,31 +19,49 @@ resource "google_bigquery_table" "test_table" { schema = < Date: Tue, 19 Mar 2024 11:27:08 +0000 Subject: [PATCH 053/169] ENH: Allow `sender_type=None` in `create_push_subscription` --- octue/cli.py | 2 +- octue/cloud/pub_sub/__init__.py | 13 +++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/octue/cli.py b/octue/cli.py index 02b5f8dee..ec1cd4b8d 100644 --- a/octue/cli.py +++ b/octue/cli.py @@ -366,7 +366,7 @@ def create_push_subscription( revision_tag, ): """Create a Google Pub/Sub push subscription for an Octue service for it to receive questions from parents. If a - corresponding topic doesn't exist, it will be created. The subscription name is printed on completion. + corresponding topic doesn't exist, it will be created first. The subscription name is printed on completion. PROJECT_NAME is the name of the Google Cloud project in which the subscription will be created diff --git a/octue/cloud/pub_sub/__init__.py b/octue/cloud/pub_sub/__init__.py index 4636f0593..62282710c 100644 --- a/octue/cloud/pub_sub/__init__.py +++ b/octue/cloud/pub_sub/__init__.py @@ -12,13 +12,13 @@ VALID_SENDER_TYPES = {PARENT_SENDER_TYPE, CHILD_SENDER_TYPE} -def create_push_subscription(project_name, sruid, push_endpoint, sender_type, expiration_time=None): - """Create a Google Pub/Sub push subscription. If a corresponding topic doesn't exist, it will be created. +def create_push_subscription(project_name, sruid, push_endpoint, sender_type=None, expiration_time=None): + """Create a Google Pub/Sub push subscription. If a corresponding topic doesn't exist, it will be created first. :param str project_name: the name of the Google Cloud project in which the subscription will be created :param str sruid: the SRUID (service revision unique identifier) :param str push_endpoint: the HTTP/HTTPS endpoint of the service to push to. It should be fully formed and include the 'https://' prefix - :param str sender_type: the type of event to subscribe to (must be one of "PARENT" or "CHILD") + :param str|None sender_type: if specified, the type of event to subscribe to (must be one of "PARENT" or "CHILD"); otherwise, no filter is applied to the subscription :param float|None expiration_time: the number of seconds of inactivity after which the subscription should expire. If not provided, no expiration time is applied to the subscription """ if sender_type not in VALID_SENDER_TYPES: @@ -34,11 +34,16 @@ def create_push_subscription(project_name, sruid, push_endpoint, sender_type, ex else: expiration_time = None + if sender_type: + subscription_filter = f'attributes.sender_type = "{sender_type}"' + else: + subscription_filter = None + subscription = Subscription( name=pub_sub_sruid, topic=topic, project_name=project_name, - filter=f'attributes.sender_type = "{sender_type}"', + filter=subscription_filter, expiration_time=expiration_time, push_endpoint=push_endpoint, ) From 06d029ecfa64901e5886c65f07447aac42a3385a Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 19 Mar 2024 11:38:24 +0000 Subject: [PATCH 054/169] ENH: Allow specifying a filter in `create-push-subscription` CLI command --- octue/cli.py | 13 +++++++++-- octue/cloud/pub_sub/__init__.py | 17 ++------------ tests/test_cli.py | 40 +++++++++++++++++++++++++++------ 3 files changed, 46 insertions(+), 24 deletions(-) diff --git a/octue/cli.py b/octue/cli.py index ec1cd4b8d..f1389456e 100644 --- a/octue/cli.py +++ b/octue/cli.py @@ -11,7 +11,7 @@ from google import auth from octue.cloud import pub_sub, storage -from octue.cloud.pub_sub.service import PARENT_SENDER_TYPE, Service +from octue.cloud.pub_sub.service import Service from octue.cloud.service_id import create_sruid, get_sruid_parts from octue.cloud.storage import GoogleCloudStorageClient from octue.configuration import load_service_and_app_configuration @@ -357,6 +357,14 @@ def deploy(): help="The service revision tag (e.g. 1.0.7). If this option isn't given, a random 'cool name' tag is generated e.g" ". 'curious-capybara'.", ) +@click.option( + "--filter", + is_flag=False, + default='attributes.sender_type = "PARENT"', + show_default=True, + help="A filter to apply to the subscription (see https://cloud.google.com/pubsub/docs/subscription-message-filter)" + ". If not provided, the default filter is applied.", +) def create_push_subscription( project_name, service_namespace, @@ -364,6 +372,7 @@ def create_push_subscription( push_endpoint, expiration_time, revision_tag, + filter, ): """Create a Google Pub/Sub push subscription for an Octue service for it to receive questions from parents. If a corresponding topic doesn't exist, it will be created first. The subscription name is printed on completion. @@ -384,7 +393,7 @@ def create_push_subscription( sruid, push_endpoint, expiration_time=expiration_time, - sender_type=PARENT_SENDER_TYPE, + subscription_filter=filter or None, ) click.echo(sruid) diff --git a/octue/cloud/pub_sub/__init__.py b/octue/cloud/pub_sub/__init__.py index 62282710c..a20532a74 100644 --- a/octue/cloud/pub_sub/__init__.py +++ b/octue/cloud/pub_sub/__init__.py @@ -7,23 +7,15 @@ __all__ = ["Subscription", "Topic"] -PARENT_SENDER_TYPE = "PARENT" -CHILD_SENDER_TYPE = "CHILD" -VALID_SENDER_TYPES = {PARENT_SENDER_TYPE, CHILD_SENDER_TYPE} - - -def create_push_subscription(project_name, sruid, push_endpoint, sender_type=None, expiration_time=None): +def create_push_subscription(project_name, sruid, push_endpoint, subscription_filter=None, expiration_time=None): """Create a Google Pub/Sub push subscription. If a corresponding topic doesn't exist, it will be created first. :param str project_name: the name of the Google Cloud project in which the subscription will be created :param str sruid: the SRUID (service revision unique identifier) :param str push_endpoint: the HTTP/HTTPS endpoint of the service to push to. It should be fully formed and include the 'https://' prefix - :param str|None sender_type: if specified, the type of event to subscribe to (must be one of "PARENT" or "CHILD"); otherwise, no filter is applied to the subscription + :param str|None subscription_filter: if specified, the filter to apply to the subscription; otherwise, no filter is applied :param float|None expiration_time: the number of seconds of inactivity after which the subscription should expire. If not provided, no expiration time is applied to the subscription """ - if sender_type not in VALID_SENDER_TYPES: - raise ValueError(f"`sender_type` must be one of {VALID_SENDER_TYPES!r}; received {sender_type!r}") - pub_sub_sruid = convert_service_id_to_pub_sub_form(sruid) topic = Topic(name=pub_sub_sruid, project_name=project_name) @@ -34,11 +26,6 @@ def create_push_subscription(project_name, sruid, push_endpoint, sender_type=Non else: expiration_time = None - if sender_type: - subscription_filter = f'attributes.sender_type = "{sender_type}"' - else: - subscription_filter = None - subscription = Subscription( name=pub_sub_sruid, topic=topic, diff --git a/tests/test_cli.py b/tests/test_cli.py index aa07c71e5..5d15f0037 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -403,7 +403,7 @@ def test_create_push_subscription(self): ): with self.subTest(expiration_time_option=expiration_time_option): with patch("octue.cloud.pub_sub.Topic", new=MockTopic): - with patch("octue.cloud.pub_sub.Subscription") as mock_subscription: + with patch("octue.cloud.pub_sub.Subscription") as subscription: result = CliRunner().invoke( octue_cli, [ @@ -420,10 +420,36 @@ def test_create_push_subscription(self): self.assertIsNone(result.exception) self.assertEqual(result.exit_code, 0) + self.assertEqual(subscription.call_args.kwargs["name"], "octue.example-service.3-5-0") + self.assertEqual(subscription.call_args.kwargs["push_endpoint"], "https://example.com/endpoint") + self.assertEqual(subscription.call_args.kwargs["expiration_time"], expected_expiration_time) + + def test_create_push_subscription_with_filter(self): + """Test that filters are added to subscriptions correctly when creating a push subscription.""" + for filter_option, expected_filter in ( + ([], 'attributes.sender_type = "PARENT"'), + (["--filter="], None), + (['--filter=attributes.sender_type = "CHILD"'], 'attributes.sender_type = "CHILD"'), + ): + with self.subTest(filter_option=filter_option): + with patch("octue.cloud.pub_sub.Topic", new=MockTopic): + with patch("octue.cloud.pub_sub.Subscription") as subscription: + result = CliRunner().invoke( + octue_cli, + [ + "deploy", + "create-push-subscription", + "my-project", + "octue", + "example-service", + "https://example.com/endpoint", + "--revision-tag=3.5.0", + *filter_option, + ], + ) - self.assertEqual(mock_subscription.call_args.kwargs["name"], "octue.example-service.3-5-0") - self.assertEqual( - mock_subscription.call_args.kwargs["push_endpoint"], - "https://example.com/endpoint", - ) - self.assertEqual(mock_subscription.call_args.kwargs["expiration_time"], expected_expiration_time) + self.assertIsNone(result.exception) + self.assertEqual(result.exit_code, 0) + self.assertEqual(subscription.call_args.kwargs["name"], "octue.example-service.3-5-0") + self.assertEqual(subscription.call_args.kwargs["push_endpoint"], "https://example.com/endpoint") + self.assertEqual(subscription.call_args.kwargs["filter"], expected_filter) From 7c8069084d53e89b7cd93f2b04e4698993e236c1 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 19 Mar 2024 11:55:36 +0000 Subject: [PATCH 055/169] DOC: Improve CLI command help message --- octue/cli.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/octue/cli.py b/octue/cli.py index f1389456e..130f8cfcf 100644 --- a/octue/cli.py +++ b/octue/cli.py @@ -362,8 +362,9 @@ def deploy(): is_flag=False, default='attributes.sender_type = "PARENT"', show_default=True, - help="A filter to apply to the subscription (see https://cloud.google.com/pubsub/docs/subscription-message-filter)" - ". If not provided, the default filter is applied.", + help="An optional filter to apply to the subscription (see " + "https://cloud.google.com/pubsub/docs/subscription-message-filter). If not provided, the default filter is applied." + " To disable filtering, provide an empty string.", ) def create_push_subscription( project_name, From c5285b383394146095635756fb29321891145fe4 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 19 Mar 2024 17:36:27 +0000 Subject: [PATCH 056/169] OPS: Use reusable workflows --- .github/workflows/python-ci.yml | 13 ++++------- .github/workflows/update-pull-request.yml | 28 ++++++++--------------- 2 files changed, 14 insertions(+), 27 deletions(-) diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 6ee800108..10911b146 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -14,15 +14,10 @@ on: jobs: check-semantic-version: if: "!contains(github.event.head_commit.message, 'skipci')" - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - uses: octue/check-semantic-version@1.0.0.beta-9 - with: - path: pyproject.toml - breaking_change_indicated_by: minor + uses: octue/workflows/.github/workflows/check-semantic-version.yml@main + with: + path: pyproject.toml + breaking_change_indicated_by: minor run-tests: if: "!contains(github.event.head_commit.message, 'skipci')" diff --git a/.github/workflows/update-pull-request.yml b/.github/workflows/update-pull-request.yml index 30f1c7150..6866ade6c 100644 --- a/.github/workflows/update-pull-request.yml +++ b/.github/workflows/update-pull-request.yml @@ -1,26 +1,18 @@ # This workflow updates the pull request description with an auto-generated section containing the categorised commit -# message headers of the pull request's commits. The auto generated section is enveloped between two comments: -# "" and "". Anything outside these in the -# description is left untouched. Auto-generated updates can be skipped for a commit if +# message headers of the commits since the last pull request merged into main. The auto generated section is enveloped +# between two comments: "" and "". Anything +# outside these in the description is left untouched. Auto-generated updates can be skipped for a commit if # "" is added to the pull request description. name: update-pull-request -on: pull_request +on: [pull_request] jobs: description: - if: "!contains(github.event.pull_request.body, '')" - runs-on: ubuntu-latest - steps: - - uses: octue/generate-pull-request-description@1.0.0.beta-2 - id: pr-description - with: - pull_request_url: ${{ github.event.pull_request.url }} - api_token: ${{ secrets.GITHUB_TOKEN }} - - - name: Update pull request body - uses: riskledger/update-pr-description@v2 - with: - body: ${{ steps.pr-description.outputs.pull_request_description }} - token: ${{ secrets.GITHUB_TOKEN }} + uses: octue/workflows/.github/workflows/generate-pull-request-description.yml@main + secrets: + token: ${{ secrets.GITHUB_TOKEN }} + permissions: + contents: read + pull-requests: write From 79732f70b4d37b3c6025629fcb3c16be31a958b4 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 19 Mar 2024 17:44:55 +0000 Subject: [PATCH 057/169] ENH: Add ability to add suffix to subscription name in CLI --- octue/cli.py | 10 ++++++++++ octue/cloud/pub_sub/__init__.py | 25 ++++++++++++++++++++----- tests/test_cli.py | 24 ++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 5 deletions(-) diff --git a/octue/cli.py b/octue/cli.py index 130f8cfcf..9d649b597 100644 --- a/octue/cli.py +++ b/octue/cli.py @@ -366,6 +366,14 @@ def deploy(): "https://cloud.google.com/pubsub/docs/subscription-message-filter). If not provided, the default filter is applied." " To disable filtering, provide an empty string.", ) +@click.option( + "--subscription-suffix", + is_flag=False, + default=None, + show_default=True, + help="An optional suffix to add to the end of the subscription name. This is useful when needing to create " + "multiple subscriptions for the same topic (subscription names are unique).", +) def create_push_subscription( project_name, service_namespace, @@ -374,6 +382,7 @@ def create_push_subscription( expiration_time, revision_tag, filter, + subscription_suffix, ): """Create a Google Pub/Sub push subscription for an Octue service for it to receive questions from parents. If a corresponding topic doesn't exist, it will be created first. The subscription name is printed on completion. @@ -395,6 +404,7 @@ def create_push_subscription( push_endpoint, expiration_time=expiration_time, subscription_filter=filter or None, + subscription_suffix=subscription_suffix, ) click.echo(sruid) diff --git a/octue/cloud/pub_sub/__init__.py b/octue/cloud/pub_sub/__init__.py index a20532a74..ab0cccf99 100644 --- a/octue/cloud/pub_sub/__init__.py +++ b/octue/cloud/pub_sub/__init__.py @@ -7,18 +7,33 @@ __all__ = ["Subscription", "Topic"] -def create_push_subscription(project_name, sruid, push_endpoint, subscription_filter=None, expiration_time=None): - """Create a Google Pub/Sub push subscription. If a corresponding topic doesn't exist, it will be created first. +def create_push_subscription( + project_name, + sruid, + push_endpoint, + subscription_filter=None, + expiration_time=None, + subscription_suffix=None, +): + """Create a Google Pub/Sub push subscription for an Octue service for it to receive questions from parents. If a + corresponding topic doesn't exist, it will be created first. :param str project_name: the name of the Google Cloud project in which the subscription will be created :param str sruid: the SRUID (service revision unique identifier) :param str push_endpoint: the HTTP/HTTPS endpoint of the service to push to. It should be fully formed and include the 'https://' prefix :param str|None subscription_filter: if specified, the filter to apply to the subscription; otherwise, no filter is applied :param float|None expiration_time: the number of seconds of inactivity after which the subscription should expire. If not provided, no expiration time is applied to the subscription + :param str|None subscription_suffix: if provided, add a suffix to the end of the subscription name. This is useful when needing to create multiple subscriptions for the same topic (subscription names are unique). + :return None: """ - pub_sub_sruid = convert_service_id_to_pub_sub_form(sruid) + topic_name = convert_service_id_to_pub_sub_form(sruid) - topic = Topic(name=pub_sub_sruid, project_name=project_name) + if subscription_suffix: + subscription_name = topic_name + subscription_suffix + else: + subscription_name = topic_name + + topic = Topic(name=topic_name, project_name=project_name) topic.create(allow_existing=True) if expiration_time: @@ -27,7 +42,7 @@ def create_push_subscription(project_name, sruid, push_endpoint, subscription_fi expiration_time = None subscription = Subscription( - name=pub_sub_sruid, + name=subscription_name, topic=topic, project_name=project_name, filter=subscription_filter, diff --git a/tests/test_cli.py b/tests/test_cli.py index 5d15f0037..7ab87ae06 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -453,3 +453,27 @@ def test_create_push_subscription_with_filter(self): self.assertEqual(subscription.call_args.kwargs["name"], "octue.example-service.3-5-0") self.assertEqual(subscription.call_args.kwargs["push_endpoint"], "https://example.com/endpoint") self.assertEqual(subscription.call_args.kwargs["filter"], expected_filter) + + def test_create_push_subscription_with_subscription_suffix(self): + """Test that subscription suffixes are added to subscription names correctly when creating a push subscription.""" + with patch("octue.cloud.pub_sub.Topic", new=MockTopic): + with patch("octue.cloud.pub_sub.Subscription") as subscription: + result = CliRunner().invoke( + octue_cli, + [ + "deploy", + "create-push-subscription", + "my-project", + "octue", + "example-service", + "https://example.com/endpoint", + "--revision-tag=3.5.0", + "--subscription-suffix=-peter-rabbit", + ], + ) + + self.assertIsNone(result.exception) + self.assertEqual(result.exit_code, 0) + self.assertEqual(subscription.call_args.kwargs["topic"].name, "octue.services.octue.example-service.3-5-0") + self.assertEqual(subscription.call_args.kwargs["name"], "octue.example-service.3-5-0-peter-rabbit") + self.assertEqual(subscription.call_args.kwargs["push_endpoint"], "https://example.com/endpoint") From b840e18a0939db2266f483053360b9f2e86f096a Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 21 Mar 2024 11:01:11 +0000 Subject: [PATCH 058/169] WIP: Add commented out pub/sub trigger for cloud function skipci --- terraform/functions.tf | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/terraform/functions.tf b/terraform/functions.tf index 962647acc..cd9d7b13f 100644 --- a/terraform/functions.tf +++ b/terraform/functions.tf @@ -23,6 +23,13 @@ resource "google_cloudfunctions2_function" "event_handler" { } } +# event_trigger { +# trigger_region = var.region +# event_type = "google.cloud.pubsub.topic.v1.messagePublished" +# pubsub_topic = google_pubsub_topic.topic.id +# retry_policy = "RETRY_POLICY_RETRY" +# } + } From 199a3398133098d5a5c0c320d51ec1da2a29b6f4 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 21 Mar 2024 11:05:26 +0000 Subject: [PATCH 059/169] OPS: Fix workflow description skipci --- .github/workflows/update-pull-request.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/update-pull-request.yml b/.github/workflows/update-pull-request.yml index 6866ade6c..d77ca388b 100644 --- a/.github/workflows/update-pull-request.yml +++ b/.github/workflows/update-pull-request.yml @@ -1,7 +1,7 @@ # This workflow updates the pull request description with an auto-generated section containing the categorised commit -# message headers of the commits since the last pull request merged into main. The auto generated section is enveloped -# between two comments: "" and "". Anything -# outside these in the description is left untouched. Auto-generated updates can be skipped for a commit if +# message headers of the pull request's commits. The auto generated section is enveloped between two comments: +# "" and "". Anything outside these in the +# description is left untouched. Auto-generated updates can be skipped for a commit if # "" is added to the pull request description. name: update-pull-request From 78ca79a0b1db44a74e491f286d4e9827bdfe0d54 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 21 Mar 2024 11:35:41 +0000 Subject: [PATCH 060/169] TST: Fix test description skipci --- tests/cloud/pub_sub/test_subscription.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cloud/pub_sub/test_subscription.py b/tests/cloud/pub_sub/test_subscription.py index 4303b8970..903247a15 100644 --- a/tests/cloud/pub_sub/test_subscription.py +++ b/tests/cloud/pub_sub/test_subscription.py @@ -137,7 +137,7 @@ def test_is_pull_subscription(self): self.assertFalse(self.subscription.is_bigquery_subscription) def test_is_push_subscription(self): - """Test that `is_push_subscription` is `True` for a pull subscription.""" + """Test that `is_push_subscription` is `True` for a push subscription.""" push_subscription = Subscription(name="world", topic=self.topic, push_endpoint="https://example.com/endpoint") self.assertTrue(push_subscription.is_push_subscription) self.assertFalse(push_subscription.is_pull_subscription) From c08dba1d7682022ca84a4ab496b114acbe4658d4 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 21 Mar 2024 12:02:27 +0000 Subject: [PATCH 061/169] REF: Improve clarity of `AbstractEventHandler` --- octue/cloud/events/handler.py | 35 ++++++++++++++++++---------- octue/cloud/events/replayer.py | 6 ++--- octue/cloud/pub_sub/event_handler.py | 12 +++++----- 3 files changed, 32 insertions(+), 21 deletions(-) diff --git a/octue/cloud/events/handler.py b/octue/cloud/events/handler.py index 918635cbc..f489612f8 100644 --- a/octue/cloud/events/handler.py +++ b/octue/cloud/events/handler.py @@ -28,8 +28,6 @@ class AbstractEventHandler: - question_uuid: str - def __init__( self, receiving_service, @@ -43,14 +41,17 @@ def __init__( self.receiving_service = receiving_service self.handle_monitor_message = handle_monitor_message self.record_events = record_events - self.child_sruid = None self.schema = schema self.only_handle_result = only_handle_result + # These are set when the first event is received. + self.question_uuid = None + self._child_sdk_version = None + self.child_sruid = None + self.waiting_events = None self.handled_events = [] self._previous_event_number = -1 - self._child_sdk_version = None self.skip_missing_events_after = skip_missing_events_after self._missing_event_detection_time = None @@ -81,19 +82,29 @@ def time_since_missing_event(self): @abc.abstractmethod def handle_events(self, *args, **kwargs): + """Handle events and return a handled "result" event once one is received. This method must be overridden but + can have any arguments. + + :return dict: the handled final result + """ pass @abc.abstractmethod - def _extract_event_and_attributes(self, event): + def _extract_event_and_attributes(self, container): + """Extract an event and its attributes from the event container. This method must be overridden. + + :param any container: the container of the event (e.g. a Pub/Sub message) + :return (any, dict): the event and its attributes (both must conform to the service communications event schema) + """ pass - def _extract_and_enqueue_event(self, event): - """Extract an event from the Pub/Sub message and add it to `self.waiting_events`. + def _extract_and_enqueue_event(self, container): + """Extract an event from its container and add it to `self.waiting_events`. - :param dict event: + :param any container: the container of the event (e.g. a Pub/Sub message) :return None: """ - event, attributes = self._extract_event_and_attributes(event) + event, attributes = self._extract_event_and_attributes(container) if not is_event_valid( event=event, @@ -132,7 +143,7 @@ def _attempt_to_handle_waiting_events(self): received yet), just return. After the missing event wait time has passed, if this set of missing events haven't arrived but subsequent ones have, skip to the earliest waiting event and continue from there. - :return any|None: either a non-`None` result from a event handler or `None` if nothing was returned by the event handlers or if the next in-order event hasn't been received yet + :return any|None: either a handled non-`None` result, or `None` if nothing was returned by the event handlers or if the next in-order event hasn't been received yet """ while self.waiting_events: try: @@ -207,7 +218,7 @@ def _handle_event(self, event): return handler(event) def _handle_delivery_acknowledgement(self, event): - """Mark the question as delivered to prevent resending it. + """Log that the question was delivered. :param dict event: :return None: @@ -235,7 +246,7 @@ def _handle_monitor_message(self, event): self.handle_monitor_message(event["data"]) def _handle_log_message(self, event): - """Deserialise the event into a log record and pass it to the local log handlers, adding [] to + """Deserialise the event into a log record and pass it to the local log handlers, adding the child's SRUID to the start of the log message. :param dict event: diff --git a/octue/cloud/events/replayer.py b/octue/cloud/events/replayer.py index 1c4e8c692..6c2ced625 100644 --- a/octue/cloud/events/replayer.py +++ b/octue/cloud/events/replayer.py @@ -39,6 +39,6 @@ def handle_events(self, events): self._earliest_waiting_event_number = min(self.waiting_events.keys()) return self._attempt_to_handle_waiting_events() - def _extract_event_and_attributes(self, event): - event["attributes"]["message_number"] = int(event["attributes"]["message_number"]) - return event["event"], event["attributes"] + def _extract_event_and_attributes(self, container): + container["attributes"]["message_number"] = int(container["attributes"]["message_number"]) + return container["event"], container["attributes"] diff --git a/octue/cloud/pub_sub/event_handler.py b/octue/cloud/pub_sub/event_handler.py index 60415a85a..096f4fe79 100644 --- a/octue/cloud/pub_sub/event_handler.py +++ b/octue/cloud/pub_sub/event_handler.py @@ -84,13 +84,13 @@ def _time_since_last_heartbeat(self): return datetime.now() - self._last_heartbeat def handle_events(self, timeout=60, maximum_heartbeat_interval=300): - """Pull events and handle them in the order they were sent until a result is returned by a event handler, - then return that result. + """Pull events and handle them in the order they were sent until a "result" event is handled then return the + handled result. :param float|None timeout: how long to wait for an answer before raising a `TimeoutError` - :param int|float maximum_heartbeat_interval: the maximum amount of time (in seconds) allowed between child heartbeats before an error is raised + :param int|float maximum_heartbeat_interval: the maximum amount of time [s] allowed between child heartbeats before an error is raised :raise TimeoutError: if the timeout is exceeded before receiving the final event - :return dict: the first result returned by a event handler + :return dict: the handled final result """ self._start_time = time.perf_counter() self.waiting_events = {} @@ -207,5 +207,5 @@ def _pull_and_enqueue_available_events(self, timeout): self._earliest_waiting_event_number = min(self.waiting_events.keys()) - def _extract_event_and_attributes(self, event): - return extract_event_and_attributes_from_pub_sub(event.message) + def _extract_event_and_attributes(self, container): + return extract_event_and_attributes_from_pub_sub(container.message) From 696ee99c874b7c607c1d8b8e9f4503878e489bbb Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 21 Mar 2024 12:07:04 +0000 Subject: [PATCH 062/169] DOC: Add docstrings to `EventReplayer` --- octue/cloud/events/replayer.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/octue/cloud/events/replayer.py b/octue/cloud/events/replayer.py index 6c2ced625..db24e3a6d 100644 --- a/octue/cloud/events/replayer.py +++ b/octue/cloud/events/replayer.py @@ -19,6 +19,16 @@ def __init__( schema=SERVICE_COMMUNICATION_SCHEMA, only_handle_result=False, ): + """A replayer for events retrieved asynchronously from some kind of storage. + + :param octue.cloud.pub_sub.service.Service receiving_service: the service that's receiving the events + :param callable|None handle_monitor_message: a function to handle monitor messages (e.g. send them to an endpoint for plotting or displaying) - this function should take a single JSON-compatible python primitive + :param bool record_events: if `True`, record received events in the `received_events` attribute + :param dict|None event_handlers: a mapping of event type names to callables that handle each type of event. The handlers should not mutate the events. + :param dict|str schema: the JSON schema (or URI of one) to validate events against + :param bool only_handle_result: if `True`, skip non-result events and only handle the result event + :return None: + """ super().__init__( receiving_service or Service(backend=ServiceBackend(), service_id="local/local:local"), handle_monitor_message=handle_monitor_message, @@ -30,6 +40,10 @@ def __init__( ) def handle_events(self, events): + """Handle the given events and return a handled "result" event if one is reached. + + :return dict: the handled final result + """ self.waiting_events = {} self._previous_event_number = -1 @@ -40,5 +54,10 @@ def handle_events(self, events): return self._attempt_to_handle_waiting_events() def _extract_event_and_attributes(self, container): + """Extract an event and its attributes from the event container. + + :param dict container: the container of the event + :return (any, dict): the event and its attributes + """ container["attributes"]["message_number"] = int(container["attributes"]["message_number"]) return container["event"], container["attributes"] From e6ef7a78747b9e7a490c8915870a015909591262 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 21 Mar 2024 12:15:11 +0000 Subject: [PATCH 063/169] REF: Rename `extract_event_and_attributes_from_pub_sub` --- octue/cloud/pub_sub/events.py | 2 +- octue/cloud/pub_sub/service.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/octue/cloud/pub_sub/events.py b/octue/cloud/pub_sub/events.py index 863b12033..7bb680253 100644 --- a/octue/cloud/pub_sub/events.py +++ b/octue/cloud/pub_sub/events.py @@ -5,7 +5,7 @@ from octue.utils.objects import getattr_or_subscribe -def extract_event_and_attributes_from_pub_sub(message): +def extract_event_and_attributes_from_pub_sub_message(message): """Extract an Octue service event and its attributes from a Google Pub/Sub message in either direct Pub/Sub format or in the Google Cloud Run format. diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index c1266e262..b250dd69f 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -17,7 +17,7 @@ from octue.cloud.events.validation import raise_if_event_is_invalid from octue.cloud.pub_sub import Subscription, Topic from octue.cloud.pub_sub.event_handler import GoogleCloudPubSubEventHandler -from octue.cloud.pub_sub.events import extract_event_and_attributes_from_pub_sub +from octue.cloud.pub_sub.events import extract_event_and_attributes_from_pub_sub_message from octue.cloud.pub_sub.logging import GoogleCloudPubSubHandler from octue.cloud.service_id import ( convert_service_id_to_pub_sub_form, @@ -581,7 +581,7 @@ def _parse_question(self, question): if hasattr(question, "ack"): question.ack() - event, attributes = extract_event_and_attributes_from_pub_sub(question) + event, attributes = extract_event_and_attributes_from_pub_sub_message(question) event_for_validation = copy.deepcopy(event) raise_if_event_is_invalid( From 74fb89741954ce62b0af9ed3e6ad25d516e743e8 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 21 Mar 2024 12:15:24 +0000 Subject: [PATCH 064/169] REF: Improve clarity of `GoogleCloudPubSubEventHandler` --- octue/cloud/pub_sub/event_handler.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/octue/cloud/pub_sub/event_handler.py b/octue/cloud/pub_sub/event_handler.py index 096f4fe79..cc9ef5fc7 100644 --- a/octue/cloud/pub_sub/event_handler.py +++ b/octue/cloud/pub_sub/event_handler.py @@ -8,7 +8,7 @@ from octue.cloud.events.handler import AbstractEventHandler from octue.cloud.events.validation import SERVICE_COMMUNICATION_SCHEMA -from octue.cloud.pub_sub.events import extract_event_and_attributes_from_pub_sub +from octue.cloud.pub_sub.events import extract_event_and_attributes_from_pub_sub_message from octue.utils.threads import RepeatingTimer @@ -20,7 +20,7 @@ class GoogleCloudPubSubEventHandler(AbstractEventHandler): - """A handler for events received as Google Pub/Sub messages from a pull subscription. + """A synchronous handler for events received as Google Pub/Sub messages from a pull subscription. :param octue.cloud.pub_sub.subscription.Subscription subscription: the subscription messages are pulled from :param octue.cloud.pub_sub.service.Service receiving_service: the service that's receiving the events @@ -53,7 +53,6 @@ def __init__( skip_missing_events_after=skip_missing_events_after, ) - self.waiting_events = None self._subscriber = SubscriberClient() self._heartbeat_checker = None self._last_heartbeat = None @@ -84,8 +83,8 @@ def _time_since_last_heartbeat(self): return datetime.now() - self._last_heartbeat def handle_events(self, timeout=60, maximum_heartbeat_interval=300): - """Pull events and handle them in the order they were sent until a "result" event is handled then return the - handled result. + """Pull events fromthe subscription and handle them in the order they were sent until a "result" event is + handled, then return the handled result. :param float|None timeout: how long to wait for an answer before raising a `TimeoutError` :param int|float maximum_heartbeat_interval: the maximum amount of time [s] allowed between child heartbeats before an error is raised @@ -208,4 +207,9 @@ def _pull_and_enqueue_available_events(self, timeout): self._earliest_waiting_event_number = min(self.waiting_events.keys()) def _extract_event_and_attributes(self, container): - return extract_event_and_attributes_from_pub_sub(container.message) + """Extract an event and its attributes from the Pub/Sub message. + + :param dict container: the container of the event + :return (any, dict): the event and its attributes + """ + return extract_event_and_attributes_from_pub_sub_message(container.message) From 1bd36fe6f604d9a69ae3e0c8f622f453de7727e5 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 21 Mar 2024 12:20:45 +0000 Subject: [PATCH 065/169] DOC: Add missing param to docstring skipci --- octue/cloud/pub_sub/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index b250dd69f..71c660f67 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -485,7 +485,7 @@ def _send_question( :param bool forward_logs: whether to request the child to forward its logs :param str save_diagnostics: must be one of {"SAVE_DIAGNOSTICS_OFF", "SAVE_DIAGNOSTICS_ON_CRASH", "SAVE_DIAGNOSTICS_ON"}; if turned on, allow the input values and manifest (and its datasets) to be saved by the child either all the time or just if it fails while processing them :param octue.cloud.pub_sub.topic.Topic topic: topic to send the acknowledgement to - :param str question_uuid: + :param str question_uuid: the UUID of the question being sent :param float timeout: time in seconds after which to give up sending :return None: """ From 00ae871c813f5239811b2c656ff719c523c5e56a Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 21 Mar 2024 14:06:43 +0000 Subject: [PATCH 066/169] REF: Replace `bigquery_table_id` argument with `asynchronous` --- octue/cloud/emulators/_pub_sub.py | 5 +-- octue/cloud/emulators/child.py | 6 ++-- octue/cloud/pub_sub/service.py | 33 ++++++++----------- octue/resources/child.py | 12 +++---- .../cloud_run/test_cloud_run_deployment.py | 14 +++----- tests/cloud/pub_sub/test_service.py | 23 +++++-------- 6 files changed, 39 insertions(+), 54 deletions(-) diff --git a/octue/cloud/emulators/_pub_sub.py b/octue/cloud/emulators/_pub_sub.py index 000006cc4..e497be5e5 100644 --- a/octue/cloud/emulators/_pub_sub.py +++ b/octue/cloud/emulators/_pub_sub.py @@ -320,7 +320,7 @@ def ask( save_diagnostics="SAVE_DIAGNOSTICS_ON_CRASH", question_uuid=None, push_endpoint=None, - bigquery_table_id=None, + asynchronous=False, timeout=86400, parent_sdk_version=importlib.metadata.version("octue"), ): @@ -336,6 +336,7 @@ def ask( :param str save_diagnostics: :param str|None question_uuid: :param str|None push_endpoint: + :param bool asynchronous: :param float|None timeout: :return MockFuture, str: """ @@ -349,7 +350,7 @@ def ask( save_diagnostics=save_diagnostics, question_uuid=question_uuid, push_endpoint=push_endpoint, - bigquery_table_id=bigquery_table_id, + asynchronous=asynchronous, timeout=timeout, ) diff --git a/octue/cloud/emulators/child.py b/octue/cloud/emulators/child.py index 8d4ab3b10..346ca4506 100644 --- a/octue/cloud/emulators/child.py +++ b/octue/cloud/emulators/child.py @@ -94,7 +94,7 @@ def ask( record_messages=True, question_uuid=None, push_endpoint=None, - bigquery_table_id=None, + asynchronous=False, timeout=86400, ): """Ask the child emulator a question and receive its emulated response messages. Unlike a real child, the input @@ -109,7 +109,7 @@ def ask( :param bool record_messages: if `True`, record messages received from the child in the `received_messages` property :param str|None question_uuid: the UUID to use for the question if a specific one is needed; a UUID is generated if not :param str|None push_endpoint: if answers to the question should be pushed to an endpoint, provide its URL here (the returned subscription will be a push subscription); if not, leave this as `None` - :param str|None bigquery_table_id: if answers to the questions should be written to BigQuery, provide the ID of the table here (e.g. "your-project.your-dataset.your-table") (the returned subscription will be a BigQuery subscription); if not, leave this as `None` + :param bool asynchronous: if `True`, don't create an answer subscription :param float timeout: time in seconds to wait for an answer before raising a timeout error :raise TimeoutError: if the timeout is exceeded while waiting for an answer :return dict: a dictionary containing the keys "output_values" and "output_manifest" @@ -125,7 +125,7 @@ def ask( allow_local_files=allow_local_files, question_uuid=question_uuid, push_endpoint=push_endpoint, - bigquery_table_id=bigquery_table_id, + asynchronous=asynchronous, ) return self._parent.wait_for_answer( diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index 71c660f67..00bcaf6ba 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -281,7 +281,7 @@ def ask( save_diagnostics="SAVE_DIAGNOSTICS_ON_CRASH", # This is repeated as a string here to avoid a circular import. question_uuid=None, push_endpoint=None, - bigquery_table_id=None, + asynchronous=False, timeout=86400, ): """Ask a child a question (i.e. send it input values for it to analyse and produce output values for) and return @@ -297,9 +297,9 @@ def ask( :param str save_diagnostics: must be one of {"SAVE_DIAGNOSTICS_OFF", "SAVE_DIAGNOSTICS_ON_CRASH", "SAVE_DIAGNOSTICS_ON"}; if turned on, allow the input values and manifest (and its datasets) to be saved by the child either all the time or just if it fails while processing them :param str|None question_uuid: the UUID to use for the question if a specific one is needed; a UUID is generated if not :param str|None push_endpoint: if answers to the question should be pushed to an endpoint, provide its URL here (the returned subscription will be a push subscription); if not, leave this as `None` - :param str|None bigquery_table_id: if answers to the questions should be written to BigQuery, provide the ID of the table here (e.g. "your-project.your-dataset.your-table") (the returned subscription will be a BigQuery subscription); if not, leave this as `None` + :param bool asynchronous: if `True`, don't create an answer subscription :param float|None timeout: time in seconds to keep retrying sending the question - :return (octue.cloud.pub_sub.subscription.Subscription, str): the answer subscription and question UUID + :return (octue.cloud.pub_sub.subscription.Subscription|None, str): the answer subscription (if the question is synchronous) and question UUID """ service_namespace, service_name, service_revision_tag = split_service_id(service_id) @@ -326,22 +326,23 @@ def ask( "the new cloud locations." ) - pub_sub_service_id = convert_service_id_to_pub_sub_form(service_id) - topic = Topic(name=pub_sub_service_id, project_name=self.backend.project_name) + topic = Topic(name=convert_service_id_to_pub_sub_form(service_id), project_name=self.backend.project_name) if not topic.exists(timeout=timeout): raise octue.exceptions.ServiceNotFound(f"Service with ID {service_id!r} cannot be found.") question_uuid = question_uuid or str(uuid.uuid4()) - answer_subscription = Subscription( - name=".".join((topic.name, ANSWERS_NAMESPACE, question_uuid)), - topic=topic, - filter=f'attributes.question_uuid = "{question_uuid}" AND attributes.sender_type = "{CHILD_SENDER_TYPE}"', - push_endpoint=push_endpoint, - bigquery_table_id=bigquery_table_id, - ) - answer_subscription.create(allow_existing=False) + if asynchronous: + answer_subscription = None + else: + answer_subscription = Subscription( + name=".".join((topic.name, ANSWERS_NAMESPACE, question_uuid)), + topic=topic, + filter=f'attributes.question_uuid = "{question_uuid}" AND attributes.sender_type = "{CHILD_SENDER_TYPE}"', + push_endpoint=push_endpoint, + ) + answer_subscription.create(allow_existing=False) self._send_question( input_values=input_values, @@ -382,12 +383,6 @@ def wait_for_answer( f"its push endpoint at {subscription.push_endpoint!r}." ) - if subscription.is_bigquery_subscription: - raise octue.exceptions.NotAPullSubscription( - f"{subscription.path!r} is a BigQuery subscription so it cannot be waited on for an answer. Please " - f"check its BigQuery table {subscription.bigquery_table_id!r}." - ) - self._event_handler = GoogleCloudPubSubEventHandler( subscription=subscription, receiving_service=self, diff --git a/octue/resources/child.py b/octue/resources/child.py index 78a32b0b0..a849240b1 100644 --- a/octue/resources/child.py +++ b/octue/resources/child.py @@ -64,15 +64,15 @@ def ask( save_diagnostics="SAVE_DIAGNOSTICS_ON_CRASH", # This is repeated as a string here to avoid a circular import. question_uuid=None, push_endpoint=None, - bigquery_table_id=None, + asynchronous=False, timeout=86400, maximum_heartbeat_interval=300, ): """Ask the child either: - A synchronous (ask-and-wait) question and wait for it to return an output. Questions are synchronous if - neither the `push_endpoint` or the `bigquery_table_id` argument is provided. + the `push_endpoint` isn't provided and `asynchronous=False`. - An asynchronous (fire-and-forget) question and return immediately. To make a question asynchronous, provide - either the `push_endpoint` or `bigquery_table_id` argument. + the `push_endpoint` argument or set `asynchronous=False`. :param any|None input_values: any input values for the question, conforming with the schema in the child's twine :param octue.resources.manifest.Manifest|None input_manifest: an input manifest of any datasets needed for the question, conforming with the schema in the child's twine @@ -84,7 +84,7 @@ def ask( :param str save_diagnostics: must be one of {"SAVE_DIAGNOSTICS_OFF", "SAVE_DIAGNOSTICS_ON_CRASH", "SAVE_DIAGNOSTICS_ON"}; if turned on, allow the input values and manifest (and its datasets) to be saved by the child either all the time or just if it fails while processing them :param str|None question_uuid: the UUID to use for the question if a specific one is needed; a UUID is generated if not :param str|None push_endpoint: if answers to the question should be pushed to an endpoint, provide its URL here (the returned subscription will be a push subscription); if not, leave this as `None` - :param str|None bigquery_table_id: if answers to the questions should be written to BigQuery, provide the ID of the table here (e.g. "your-project.your-dataset.your-table") (the returned subscription will be a BigQuery subscription); if not, leave this as `None` + :param bool asynchronous: if `True`, don't create an answer subscription :param float timeout: time in seconds to wait for an answer before raising a timeout error :param float|int maximum_heartbeat_interval: the maximum amount of time (in seconds) allowed between child heartbeats before an error is raised :raise TimeoutError: if the timeout is exceeded while waiting for an answer @@ -100,11 +100,11 @@ def ask( save_diagnostics=save_diagnostics, question_uuid=question_uuid, push_endpoint=push_endpoint, - bigquery_table_id=bigquery_table_id, + asynchronous=asynchronous, timeout=timeout, ) - if push_endpoint or bigquery_table_id: + if push_endpoint or asynchronous: return None return self._service.wait_for_answer( diff --git a/tests/cloud/deployment/google/cloud_run/test_cloud_run_deployment.py b/tests/cloud/deployment/google/cloud_run/test_cloud_run_deployment.py index 23a91297e..4006e9333 100644 --- a/tests/cloud/deployment/google/cloud_run/test_cloud_run_deployment.py +++ b/tests/cloud/deployment/google/cloud_run/test_cloud_run_deployment.py @@ -17,12 +17,12 @@ class TestCloudRunDeployment(TestCase): backend={"name": "GCPPubSubBackend", "project_name": os.environ["TEST_PROJECT_NAME"]}, ) - def test_cloud_run_deployment_forwards_exceptions_to_asking_service(self): + def test_forwards_exceptions_to_parent(self): """Test that exceptions raised in the (remote) responding service are forwarded to and raised by the asker.""" with self.assertRaises(twined.exceptions.InvalidValuesContents): self.child.ask(input_values={"invalid_input_data": "hello"}) - def test_cloud_run_deployment(self): + def test_synchronous_question(self): """Test that the Google Cloud Run example deployment works, providing a service that can be asked questions and send responses. """ @@ -35,11 +35,7 @@ def test_cloud_run_deployment(self): with answer["output_manifest"].datasets["example_dataset"].files.one() as (datafile, f): self.assertEqual(f.read(), "This is some example service output.") - def test_cloud_run_deployment_asynchronously(self): - """Test asking an asynchronous (BigQuery) question.""" - answer = self.child.ask( - input_values={"n_iterations": 3}, - bigquery_table_id="octue-sdk-python.octue_sdk_python_test_dataset.question-events", - ) - + def test_asynchronous_question(self): + """Test asking an asynchronous question.""" + answer = self.child.ask(input_values={"n_iterations": 3}, asynchronous=True) self.assertIsNone(answer) diff --git a/tests/cloud/pub_sub/test_service.py b/tests/cloud/pub_sub/test_service.py index e72f51419..37adc3599 100644 --- a/tests/cloud/pub_sub/test_service.py +++ b/tests/cloud/pub_sub/test_service.py @@ -181,21 +181,14 @@ def test_error_raised_if_attempting_to_wait_for_answer_from_non_pull_subscriptio """Test that an error is raised if attempting to wait for an answer from a push subscription.""" service = Service(backend=BACKEND) - for subscription in [ - MockSubscription( - name="world", - topic=MockTopic(name="world", project_name=TEST_PROJECT_NAME), - push_endpoint="https://example.com/endpoint", - ), - MockSubscription( - name="world", - topic=MockTopic(name="world", project_name=TEST_PROJECT_NAME), - bigquery_table_id="some-table", - ), - ]: - with self.subTest(subscription=subscription): - with self.assertRaises(exceptions.NotAPullSubscription): - service.wait_for_answer(subscription=subscription) + subscription = MockSubscription( + name="world", + topic=MockTopic(name="world", project_name=TEST_PROJECT_NAME), + push_endpoint="https://example.com/endpoint", + ) + + with self.assertRaises(exceptions.NotAPullSubscription): + service.wait_for_answer(subscription=subscription) def test_exceptions_in_responder_are_handled_and_sent_to_asker(self): """Test that exceptions raised in the child service are handled and sent back to the asker.""" From 49e6af4eb6a94a27572cda20440910125a17d8a2 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 21 Mar 2024 14:12:52 +0000 Subject: [PATCH 067/169] DOC: Add class-string to `AbstractEventHandler` skipci --- octue/cloud/events/handler.py | 13 +++++++++++++ octue/cloud/events/replayer.py | 21 +++++++++++---------- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/octue/cloud/events/handler.py b/octue/cloud/events/handler.py index f489612f8..c54d44b6b 100644 --- a/octue/cloud/events/handler.py +++ b/octue/cloud/events/handler.py @@ -28,6 +28,19 @@ class AbstractEventHandler: + """An abstract event handler. Inherit from this and add the `handle_events` and `_extract_event_and_attributes` + methods to handle events from a specific source synchronously or asynchronously. + + :param octue.cloud.pub_sub.service.Service receiving_service: the service that's receiving the events + :param callable|None handle_monitor_message: a function to handle monitor messages (e.g. send them to an endpoint for plotting or displaying) - this function should take a single JSON-compatible python primitive + :param bool record_events: if `True`, record received events in the `received_events` attribute + :param dict|None event_handlers: a mapping of event type names to callables that handle each type of event. The handlers should not mutate the events. + :param dict|str schema: the JSON schema (or URI of one) to validate events against + :param int|float skip_missing_events_after: the number of seconds after which to skip any events if they haven't arrived but subsequent events have + :param bool only_handle_result: if `True`, skip non-result events and only handle the result event + :return None: + """ + def __init__( self, receiving_service, diff --git a/octue/cloud/events/replayer.py b/octue/cloud/events/replayer.py index db24e3a6d..f9b055119 100644 --- a/octue/cloud/events/replayer.py +++ b/octue/cloud/events/replayer.py @@ -10,6 +10,17 @@ class EventReplayer(AbstractEventHandler): + """A replayer for events retrieved asynchronously from some kind of storage. + + :param octue.cloud.pub_sub.service.Service receiving_service: the service that's receiving the events + :param callable|None handle_monitor_message: a function to handle monitor messages (e.g. send them to an endpoint for plotting or displaying) - this function should take a single JSON-compatible python primitive + :param bool record_events: if `True`, record received events in the `received_events` attribute + :param dict|None event_handlers: a mapping of event type names to callables that handle each type of event. The handlers should not mutate the events. + :param dict|str schema: the JSON schema (or URI of one) to validate events against + :param bool only_handle_result: if `True`, skip non-result events and only handle the result event + :return None: + """ + def __init__( self, receiving_service=None, @@ -19,16 +30,6 @@ def __init__( schema=SERVICE_COMMUNICATION_SCHEMA, only_handle_result=False, ): - """A replayer for events retrieved asynchronously from some kind of storage. - - :param octue.cloud.pub_sub.service.Service receiving_service: the service that's receiving the events - :param callable|None handle_monitor_message: a function to handle monitor messages (e.g. send them to an endpoint for plotting or displaying) - this function should take a single JSON-compatible python primitive - :param bool record_events: if `True`, record received events in the `received_events` attribute - :param dict|None event_handlers: a mapping of event type names to callables that handle each type of event. The handlers should not mutate the events. - :param dict|str schema: the JSON schema (or URI of one) to validate events against - :param bool only_handle_result: if `True`, skip non-result events and only handle the result event - :return None: - """ super().__init__( receiving_service or Service(backend=ServiceBackend(), service_id="local/local:local"), handle_monitor_message=handle_monitor_message, From 86632cf50341bbba38ca1302c75fee46d68cdc58 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 21 Mar 2024 14:16:29 +0000 Subject: [PATCH 068/169] REF: Move pub/sub event handler into `octue.cloud.pub_sub.events` --- octue/cloud/emulators/child.py | 2 +- octue/cloud/pub_sub/event_handler.py | 215 ------------------ octue/cloud/pub_sub/events.py | 212 +++++++++++++++++ octue/cloud/pub_sub/service.py | 3 +- .../{test_event_handler.py => test_events.py} | 32 +-- 5 files changed, 230 insertions(+), 234 deletions(-) delete mode 100644 octue/cloud/pub_sub/event_handler.py rename tests/cloud/pub_sub/{test_event_handler.py => test_events.py} (94%) diff --git a/octue/cloud/emulators/child.py b/octue/cloud/emulators/child.py index 346ca4506..a843d55ee 100644 --- a/octue/cloud/emulators/child.py +++ b/octue/cloud/emulators/child.py @@ -322,7 +322,7 @@ def __init__(self): patches=[ patch("octue.cloud.pub_sub.service.Topic", new=MockTopic), patch("octue.cloud.pub_sub.service.Subscription", new=MockSubscription), - patch("octue.cloud.pub_sub.event_handler.SubscriberClient", new=MockSubscriber), + patch("octue.cloud.pub_sub.events.SubscriberClient", new=MockSubscriber), patch("google.cloud.pubsub_v1.SubscriberClient", new=MockSubscriber), ] ) diff --git a/octue/cloud/pub_sub/event_handler.py b/octue/cloud/pub_sub/event_handler.py deleted file mode 100644 index cc9ef5fc7..000000000 --- a/octue/cloud/pub_sub/event_handler.py +++ /dev/null @@ -1,215 +0,0 @@ -import importlib.metadata -import logging -import time -from datetime import datetime, timedelta - -from google.api_core import retry -from google.cloud.pubsub_v1 import SubscriberClient - -from octue.cloud.events.handler import AbstractEventHandler -from octue.cloud.events.validation import SERVICE_COMMUNICATION_SCHEMA -from octue.cloud.pub_sub.events import extract_event_and_attributes_from_pub_sub_message -from octue.utils.threads import RepeatingTimer - - -logger = logging.getLogger(__name__) - - -MAX_SIMULTANEOUS_MESSAGES_PULL = 50 -PARENT_SDK_VERSION = importlib.metadata.version("octue") - - -class GoogleCloudPubSubEventHandler(AbstractEventHandler): - """A synchronous handler for events received as Google Pub/Sub messages from a pull subscription. - - :param octue.cloud.pub_sub.subscription.Subscription subscription: the subscription messages are pulled from - :param octue.cloud.pub_sub.service.Service receiving_service: the service that's receiving the events - :param callable|None handle_monitor_message: a function to handle monitor messages (e.g. send them to an endpoint for plotting or displaying) - this function should take a single JSON-compatible python primitive - :param bool record_events: if `True`, record received events in the `received_events` attribute - :param dict|None event_handlers: a mapping of event type names to callables that handle each type of event. The handlers should not mutate the events. - :param dict|str schema: the JSON schema (or URI of one) to validate events against - :param int|float skip_missing_events_after: the number of seconds after which to skip any events if they haven't arrived but subsequent events have - :return None: - """ - - def __init__( - self, - subscription, - receiving_service, - handle_monitor_message=None, - record_events=True, - event_handlers=None, - schema=SERVICE_COMMUNICATION_SCHEMA, - skip_missing_events_after=10, - ): - self.subscription = subscription - - super().__init__( - receiving_service, - handle_monitor_message=handle_monitor_message, - record_events=record_events, - event_handlers=event_handlers, - schema=schema, - skip_missing_events_after=skip_missing_events_after, - ) - - self._subscriber = SubscriberClient() - self._heartbeat_checker = None - self._last_heartbeat = None - self._alive = True - self._start_time = None - - @property - def total_run_time(self): - """Get the amount of time elapsed since `self.handle_events` was called. If it hasn't been called yet, it will - be `None`. - - :return float|None: the amount of time since `self.handle_events` was called (in seconds) - """ - if self._start_time is None: - return None - - return time.perf_counter() - self._start_time - - @property - def _time_since_last_heartbeat(self): - """Get the time period since the last heartbeat was received. - - :return datetime.timedelta|None: - """ - if not self._last_heartbeat: - return None - - return datetime.now() - self._last_heartbeat - - def handle_events(self, timeout=60, maximum_heartbeat_interval=300): - """Pull events fromthe subscription and handle them in the order they were sent until a "result" event is - handled, then return the handled result. - - :param float|None timeout: how long to wait for an answer before raising a `TimeoutError` - :param int|float maximum_heartbeat_interval: the maximum amount of time [s] allowed between child heartbeats before an error is raised - :raise TimeoutError: if the timeout is exceeded before receiving the final event - :return dict: the handled final result - """ - self._start_time = time.perf_counter() - self.waiting_events = {} - self._previous_event_number = -1 - - self._heartbeat_checker = RepeatingTimer( - interval=maximum_heartbeat_interval, - function=self._monitor_heartbeat, - kwargs={"maximum_heartbeat_interval": maximum_heartbeat_interval}, - ) - - try: - self._heartbeat_checker.daemon = True - self._heartbeat_checker.start() - - while self._alive: - pull_timeout = self._check_timeout_and_get_pull_timeout(timeout) - self._pull_and_enqueue_available_events(timeout=pull_timeout) - result = self._attempt_to_handle_waiting_events() - - if result is not None: - return result - - finally: - self._heartbeat_checker.cancel() - self._subscriber.close() - - raise TimeoutError( - f"No heartbeat has been received within the maximum allowed interval of {maximum_heartbeat_interval}s." - ) - - def _monitor_heartbeat(self, maximum_heartbeat_interval): - """Change the alive status to `False` and cancel the heartbeat checker if a heartbeat hasn't been received - within the maximum allowed time interval measured from the moment of calling. - - :param float|int maximum_heartbeat_interval: the maximum amount of time (in seconds) allowed between child heartbeats without raising an error - :return None: - """ - maximum_heartbeat_interval = timedelta(seconds=maximum_heartbeat_interval) - - if self._last_heartbeat and self._time_since_last_heartbeat <= maximum_heartbeat_interval: - self._alive = True - return - - self._alive = False - self._heartbeat_checker.cancel() - - def _check_timeout_and_get_pull_timeout(self, timeout): - """Check if the timeout has been exceeded and, if it hasn't, return the timeout for the next message pull. If - the timeout has been exceeded, raise an error. - - :param int|float|None timeout: the timeout for handling all messages - :raise TimeoutError: if the timeout has been exceeded - :return int|float: the timeout for the next message pull in seconds - """ - if timeout is None: - return None - - # Get the total run time once in case it's very close to the timeout - this rules out a negative pull timeout - # being returned below. - total_run_time = self.total_run_time - - if total_run_time > timeout: - raise TimeoutError( - f"No final answer received from topic {self.subscription.topic.path!r} after {timeout} seconds." - ) - - return timeout - total_run_time - - def _pull_and_enqueue_available_events(self, timeout): - """Pull as many events from the subscription as are available and enqueue them in `self.waiting_events`, - raising a `TimeoutError` if the timeout is exceeded before succeeding. - - :param float|None timeout: how long to wait in seconds for the event before raising a `TimeoutError` - :raise TimeoutError|concurrent.futures.TimeoutError: if the timeout is exceeded - :return None: - """ - pull_start_time = time.perf_counter() - attempt = 1 - - while self._alive: - logger.debug("Pulling events from Google Pub/Sub: attempt %d.", attempt) - - pull_response = self._subscriber.pull( - request={"subscription": self.subscription.path, "max_messages": MAX_SIMULTANEOUS_MESSAGES_PULL}, - retry=retry.Retry(), - ) - - if len(pull_response.received_messages) > 0: - break - else: - logger.debug("Google Pub/Sub pull response timed out early.") - attempt += 1 - - pull_run_time = time.perf_counter() - pull_start_time - - if timeout is not None and pull_run_time > timeout: - raise TimeoutError( - f"No message received from topic {self.subscription.topic.path!r} after {timeout} seconds.", - ) - - if not pull_response.received_messages: - return - - self._subscriber.acknowledge( - request={ - "subscription": self.subscription.path, - "ack_ids": [message.ack_id for message in pull_response.received_messages], - } - ) - - for event in pull_response.received_messages: - self._extract_and_enqueue_event(event) - - self._earliest_waiting_event_number = min(self.waiting_events.keys()) - - def _extract_event_and_attributes(self, container): - """Extract an event and its attributes from the Pub/Sub message. - - :param dict container: the container of the event - :return (any, dict): the event and its attributes - """ - return extract_event_and_attributes_from_pub_sub_message(container.message) diff --git a/octue/cloud/pub_sub/events.py b/octue/cloud/pub_sub/events.py index 7bb680253..cb515d5e5 100644 --- a/octue/cloud/pub_sub/events.py +++ b/octue/cloud/pub_sub/events.py @@ -1,8 +1,24 @@ import base64 +import importlib.metadata import json +import logging +import time +from datetime import datetime, timedelta +from google.api_core import retry +from google.cloud.pubsub_v1 import SubscriberClient + +from octue.cloud.events.handler import AbstractEventHandler +from octue.cloud.events.validation import SERVICE_COMMUNICATION_SCHEMA from octue.utils.decoders import OctueJSONDecoder from octue.utils.objects import getattr_or_subscribe +from octue.utils.threads import RepeatingTimer + + +logger = logging.getLogger(__name__) + +MAX_SIMULTANEOUS_MESSAGES_PULL = 50 +PARENT_SDK_VERSION = importlib.metadata.version("octue") def extract_event_and_attributes_from_pub_sub_message(message): @@ -37,3 +53,199 @@ def extract_event_and_attributes_from_pub_sub_message(message): event = json.loads(base64.b64decode(message["data"]).decode("utf-8").strip(), cls=OctueJSONDecoder) return event, converted_attributes + + +class GoogleCloudPubSubEventHandler(AbstractEventHandler): + """A synchronous handler for events received as Google Pub/Sub messages from a pull subscription. + + :param octue.cloud.pub_sub.subscription.Subscription subscription: the subscription messages are pulled from + :param octue.cloud.pub_sub.service.Service receiving_service: the service that's receiving the events + :param callable|None handle_monitor_message: a function to handle monitor messages (e.g. send them to an endpoint for plotting or displaying) - this function should take a single JSON-compatible python primitive + :param bool record_events: if `True`, record received events in the `received_events` attribute + :param dict|None event_handlers: a mapping of event type names to callables that handle each type of event. The handlers should not mutate the events. + :param dict|str schema: the JSON schema (or URI of one) to validate events against + :param int|float skip_missing_events_after: the number of seconds after which to skip any events if they haven't arrived but subsequent events have + :return None: + """ + + def __init__( + self, + subscription, + receiving_service, + handle_monitor_message=None, + record_events=True, + event_handlers=None, + schema=SERVICE_COMMUNICATION_SCHEMA, + skip_missing_events_after=10, + ): + self.subscription = subscription + + super().__init__( + receiving_service, + handle_monitor_message=handle_monitor_message, + record_events=record_events, + event_handlers=event_handlers, + schema=schema, + skip_missing_events_after=skip_missing_events_after, + ) + + self._subscriber = SubscriberClient() + self._heartbeat_checker = None + self._last_heartbeat = None + self._alive = True + self._start_time = None + + @property + def total_run_time(self): + """Get the amount of time elapsed since `self.handle_events` was called. If it hasn't been called yet, it will + be `None`. + + :return float|None: the amount of time since `self.handle_events` was called (in seconds) + """ + if self._start_time is None: + return None + + return time.perf_counter() - self._start_time + + @property + def _time_since_last_heartbeat(self): + """Get the time period since the last heartbeat was received. + + :return datetime.timedelta|None: + """ + if not self._last_heartbeat: + return None + + return datetime.now() - self._last_heartbeat + + def handle_events(self, timeout=60, maximum_heartbeat_interval=300): + """Pull events fromthe subscription and handle them in the order they were sent until a "result" event is + handled, then return the handled result. + + :param float|None timeout: how long to wait for an answer before raising a `TimeoutError` + :param int|float maximum_heartbeat_interval: the maximum amount of time [s] allowed between child heartbeats before an error is raised + :raise TimeoutError: if the timeout is exceeded before receiving the final event + :return dict: the handled final result + """ + self._start_time = time.perf_counter() + self.waiting_events = {} + self._previous_event_number = -1 + + self._heartbeat_checker = RepeatingTimer( + interval=maximum_heartbeat_interval, + function=self._monitor_heartbeat, + kwargs={"maximum_heartbeat_interval": maximum_heartbeat_interval}, + ) + + try: + self._heartbeat_checker.daemon = True + self._heartbeat_checker.start() + + while self._alive: + pull_timeout = self._check_timeout_and_get_pull_timeout(timeout) + self._pull_and_enqueue_available_events(timeout=pull_timeout) + result = self._attempt_to_handle_waiting_events() + + if result is not None: + return result + + finally: + self._heartbeat_checker.cancel() + self._subscriber.close() + + raise TimeoutError( + f"No heartbeat has been received within the maximum allowed interval of {maximum_heartbeat_interval}s." + ) + + def _monitor_heartbeat(self, maximum_heartbeat_interval): + """Change the alive status to `False` and cancel the heartbeat checker if a heartbeat hasn't been received + within the maximum allowed time interval measured from the moment of calling. + + :param float|int maximum_heartbeat_interval: the maximum amount of time (in seconds) allowed between child heartbeats without raising an error + :return None: + """ + maximum_heartbeat_interval = timedelta(seconds=maximum_heartbeat_interval) + + if self._last_heartbeat and self._time_since_last_heartbeat <= maximum_heartbeat_interval: + self._alive = True + return + + self._alive = False + self._heartbeat_checker.cancel() + + def _check_timeout_and_get_pull_timeout(self, timeout): + """Check if the timeout has been exceeded and, if it hasn't, return the timeout for the next message pull. If + the timeout has been exceeded, raise an error. + + :param int|float|None timeout: the timeout for handling all messages + :raise TimeoutError: if the timeout has been exceeded + :return int|float: the timeout for the next message pull in seconds + """ + if timeout is None: + return None + + # Get the total run time once in case it's very close to the timeout - this rules out a negative pull timeout + # being returned below. + total_run_time = self.total_run_time + + if total_run_time > timeout: + raise TimeoutError( + f"No final answer received from topic {self.subscription.topic.path!r} after {timeout} seconds." + ) + + return timeout - total_run_time + + def _pull_and_enqueue_available_events(self, timeout): + """Pull as many events from the subscription as are available and enqueue them in `self.waiting_events`, + raising a `TimeoutError` if the timeout is exceeded before succeeding. + + :param float|None timeout: how long to wait in seconds for the event before raising a `TimeoutError` + :raise TimeoutError|concurrent.futures.TimeoutError: if the timeout is exceeded + :return None: + """ + pull_start_time = time.perf_counter() + attempt = 1 + + while self._alive: + logger.debug("Pulling events from Google Pub/Sub: attempt %d.", attempt) + + pull_response = self._subscriber.pull( + request={"subscription": self.subscription.path, "max_messages": MAX_SIMULTANEOUS_MESSAGES_PULL}, + retry=retry.Retry(), + ) + + if len(pull_response.received_messages) > 0: + break + else: + logger.debug("Google Pub/Sub pull response timed out early.") + attempt += 1 + + pull_run_time = time.perf_counter() - pull_start_time + + if timeout is not None and pull_run_time > timeout: + raise TimeoutError( + f"No message received from topic {self.subscription.topic.path!r} after {timeout} seconds.", + ) + + if not pull_response.received_messages: + return + + self._subscriber.acknowledge( + request={ + "subscription": self.subscription.path, + "ack_ids": [message.ack_id for message in pull_response.received_messages], + } + ) + + for event in pull_response.received_messages: + self._extract_and_enqueue_event(event) + + self._earliest_waiting_event_number = min(self.waiting_events.keys()) + + def _extract_event_and_attributes(self, container): + """Extract an event and its attributes from the Pub/Sub message. + + :param dict container: the container of the event + :return (any, dict): the event and its attributes + """ + return extract_event_and_attributes_from_pub_sub_message(container.message) diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index 00bcaf6ba..2827348e5 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -16,8 +16,7 @@ import octue.exceptions from octue.cloud.events.validation import raise_if_event_is_invalid from octue.cloud.pub_sub import Subscription, Topic -from octue.cloud.pub_sub.event_handler import GoogleCloudPubSubEventHandler -from octue.cloud.pub_sub.events import extract_event_and_attributes_from_pub_sub_message +from octue.cloud.pub_sub.events import GoogleCloudPubSubEventHandler, extract_event_and_attributes_from_pub_sub_message from octue.cloud.pub_sub.logging import GoogleCloudPubSubHandler from octue.cloud.service_id import ( convert_service_id_to_pub_sub_form, diff --git a/tests/cloud/pub_sub/test_event_handler.py b/tests/cloud/pub_sub/test_events.py similarity index 94% rename from tests/cloud/pub_sub/test_event_handler.py rename to tests/cloud/pub_sub/test_events.py index 8047a225b..2f97ba897 100644 --- a/tests/cloud/pub_sub/test_event_handler.py +++ b/tests/cloud/pub_sub/test_events.py @@ -12,7 +12,7 @@ MockTopic, ) from octue.cloud.emulators.child import ServicePatcher -from octue.cloud.pub_sub.event_handler import GoogleCloudPubSubEventHandler +from octue.cloud.pub_sub.events import GoogleCloudPubSubEventHandler from octue.resources.service_backends import GCPPubSubBackend from tests import TEST_PROJECT_NAME from tests.base import BaseTestCase @@ -39,7 +39,7 @@ def test_timeout(self): """Test that a TimeoutError is raised if message handling takes longer than the given timeout.""" question_uuid, _, mock_subscription = create_mock_topic_and_subscription() - with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): + with patch("octue.cloud.pub_sub.events.SubscriberClient", MockSubscriber): event_handler = GoogleCloudPubSubEventHandler( subscription=mock_subscription, receiving_service=parent, @@ -54,7 +54,7 @@ def test_in_order_messages_are_handled_in_order(self): """Test that messages received in order are handled in order.""" question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() - with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): + with patch("octue.cloud.pub_sub.events.SubscriberClient", MockSubscriber): event_handler = GoogleCloudPubSubEventHandler( subscription=mock_subscription, receiving_service=parent, @@ -86,7 +86,7 @@ def test_out_of_order_messages_are_handled_in_order(self): """Test that messages received out of order are handled in order.""" question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() - with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): + with patch("octue.cloud.pub_sub.events.SubscriberClient", MockSubscriber): event_handler = GoogleCloudPubSubEventHandler( subscription=mock_subscription, receiving_service=parent, @@ -139,7 +139,7 @@ def test_out_of_order_messages_with_end_message_first_are_handled_in_order(self) """ question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() - with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): + with patch("octue.cloud.pub_sub.events.SubscriberClient", MockSubscriber): event_handler = GoogleCloudPubSubEventHandler( subscription=mock_subscription, receiving_service=parent, @@ -190,7 +190,7 @@ def test_no_timeout(self): """Test that message handling works with no timeout.""" question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() - with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): + with patch("octue.cloud.pub_sub.events.SubscriberClient", MockSubscriber): event_handler = GoogleCloudPubSubEventHandler( subscription=mock_subscription, receiving_service=parent, @@ -230,7 +230,7 @@ def test_delivery_acknowledgement(self): """Test that a delivery acknowledgement message is handled correctly.""" question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() - with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): + with patch("octue.cloud.pub_sub.events.SubscriberClient", MockSubscriber): event_handler = GoogleCloudPubSubEventHandler(subscription=mock_subscription, receiving_service=parent) child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) @@ -256,7 +256,7 @@ def test_error_raised_if_heartbeat_not_received_before_checked(self): """Test that an error is raised if a heartbeat isn't received before a heartbeat is first checked for.""" question_uuid, _, mock_subscription = create_mock_topic_and_subscription() - with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): + with patch("octue.cloud.pub_sub.events.SubscriberClient", MockSubscriber): event_handler = GoogleCloudPubSubEventHandler(subscription=mock_subscription, receiving_service=parent) with self.assertRaises(TimeoutError) as error: @@ -269,7 +269,7 @@ def test_error_raised_if_heartbeats_stop_being_received(self): """Test that an error is raised if heartbeats stop being received within the maximum interval.""" question_uuid, _, mock_subscription = create_mock_topic_and_subscription() - with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): + with patch("octue.cloud.pub_sub.events.SubscriberClient", MockSubscriber): event_handler = GoogleCloudPubSubEventHandler(subscription=mock_subscription, receiving_service=parent) event_handler._last_heartbeat = datetime.datetime.now() - datetime.timedelta(seconds=30) @@ -283,7 +283,7 @@ def test_error_not_raised_if_heartbeat_has_been_received_in_maximum_allowed_inte """Test that an error is not raised if a heartbeat has been received in the maximum allowed interval.""" question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() - with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): + with patch("octue.cloud.pub_sub.events.SubscriberClient", MockSubscriber): event_handler = GoogleCloudPubSubEventHandler(subscription=mock_subscription, receiving_service=parent) event_handler._last_heartbeat = datetime.datetime.now() @@ -308,7 +308,7 @@ def test_error_not_raised_if_heartbeat_has_been_received_in_maximum_allowed_inte child._send_message(message=message["event"], attributes=message["attributes"], topic=mock_topic) with patch( - "octue.cloud.pub_sub.event_handler.GoogleCloudPubSubEventHandler._time_since_last_heartbeat", + "octue.cloud.pub_sub.events.GoogleCloudPubSubEventHandler._time_since_last_heartbeat", datetime.timedelta(seconds=0), ): event_handler.handle_events(maximum_heartbeat_interval=0) @@ -317,7 +317,7 @@ def test_time_since_last_heartbeat_is_none_if_no_heartbeat_received_yet(self): """Test that the time since the last heartbeat is `None` if no heartbeat has been received yet.""" question_uuid, _, mock_subscription = create_mock_topic_and_subscription() - with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): + with patch("octue.cloud.pub_sub.events.SubscriberClient", MockSubscriber): event_handler = GoogleCloudPubSubEventHandler(subscription=mock_subscription, receiving_service=parent) self.assertIsNone(event_handler._time_since_last_heartbeat) @@ -342,7 +342,7 @@ def test_missing_messages_at_start_can_be_skipped(self): """ question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() - with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): + with patch("octue.cloud.pub_sub.events.SubscriberClient", MockSubscriber): event_handler = GoogleCloudPubSubEventHandler( subscription=mock_subscription, receiving_service=parent, @@ -395,7 +395,7 @@ def test_missing_messages_in_middle_can_skipped(self): """Test that missing messages in the middle of the event stream can be skipped.""" question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() - with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): + with patch("octue.cloud.pub_sub.events.SubscriberClient", MockSubscriber): event_handler = GoogleCloudPubSubEventHandler( subscription=mock_subscription, receiving_service=parent, @@ -452,7 +452,7 @@ def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): """Test that multiple blocks of missing messages in the middle of the event stream can be skipped.""" question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() - with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): + with patch("octue.cloud.pub_sub.events.SubscriberClient", MockSubscriber): event_handler = GoogleCloudPubSubEventHandler( subscription=mock_subscription, receiving_service=parent, @@ -539,7 +539,7 @@ def test_all_messages_missing_apart_from_result(self): """Test that the result message is still handled if all other messages are missing.""" question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() - with patch("octue.cloud.pub_sub.event_handler.SubscriberClient", MockSubscriber): + with patch("octue.cloud.pub_sub.events.SubscriberClient", MockSubscriber): event_handler = GoogleCloudPubSubEventHandler( subscription=mock_subscription, receiving_service=parent, From 0eb1dfea4612246db6ae4cd127e6ec2e731a876a Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 21 Mar 2024 14:23:21 +0000 Subject: [PATCH 069/169] FIX: Ensure push subscription always returned from `Service.ask` skipci --- octue/cloud/pub_sub/service.py | 6 +++--- octue/resources/child.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index 2827348e5..e7ed3c349 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -296,9 +296,9 @@ def ask( :param str save_diagnostics: must be one of {"SAVE_DIAGNOSTICS_OFF", "SAVE_DIAGNOSTICS_ON_CRASH", "SAVE_DIAGNOSTICS_ON"}; if turned on, allow the input values and manifest (and its datasets) to be saved by the child either all the time or just if it fails while processing them :param str|None question_uuid: the UUID to use for the question if a specific one is needed; a UUID is generated if not :param str|None push_endpoint: if answers to the question should be pushed to an endpoint, provide its URL here (the returned subscription will be a push subscription); if not, leave this as `None` - :param bool asynchronous: if `True`, don't create an answer subscription + :param bool asynchronous: if `True` and not using a push endpoint, don't create an answer subscription :param float|None timeout: time in seconds to keep retrying sending the question - :return (octue.cloud.pub_sub.subscription.Subscription|None, str): the answer subscription (if the question is synchronous) and question UUID + :return (octue.cloud.pub_sub.subscription.Subscription|None, str): the answer subscription (if the question is synchronous or a push endpoint was used) and question UUID """ service_namespace, service_name, service_revision_tag = split_service_id(service_id) @@ -332,7 +332,7 @@ def ask( question_uuid = question_uuid or str(uuid.uuid4()) - if asynchronous: + if asynchronous and not push_endpoint: answer_subscription = None else: answer_subscription = Subscription( diff --git a/octue/resources/child.py b/octue/resources/child.py index a849240b1..915490fa5 100644 --- a/octue/resources/child.py +++ b/octue/resources/child.py @@ -88,7 +88,7 @@ def ask( :param float timeout: time in seconds to wait for an answer before raising a timeout error :param float|int maximum_heartbeat_interval: the maximum amount of time (in seconds) allowed between child heartbeats before an error is raised :raise TimeoutError: if the timeout is exceeded while waiting for an answer - :return dict|None: for a synchronous question, a dictionary containing the keys "output_values" and "output_manifest"; for an asynchronous question, `None` + :return dict|None: for a synchronous question, a dictionary containing the keys "output_values" and "output_manifest" from the result; for an asynchronous question, `None` """ subscription, _ = self._service.ask( service_id=self.id, From 811cc9458554fa21cac3af9d7f4b0ec730798d96 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 21 Mar 2024 14:25:43 +0000 Subject: [PATCH 070/169] TST: Remove ordering from test pass criteria for `Manifest.download` --- tests/resources/test_manifest.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/resources/test_manifest.py b/tests/resources/test_manifest.py index c59a2e823..00ce48fd9 100644 --- a/tests/resources/test_manifest.py +++ b/tests/resources/test_manifest.py @@ -313,7 +313,7 @@ def test_download_without_paths(self): """Test downloading a manifest without specifying the `paths` argument.""" manifest = Manifest(datasets={"my_dataset": self.create_nested_cloud_dataset()}) paths = manifest.download() - self.assertEqual(os.listdir(paths["my_dataset"]), ["file_1.txt", "file_0.txt", "sub-directory"]) + self.assertEqual(set(os.listdir(paths["my_dataset"])), {"file_1.txt", "file_0.txt", "sub-directory"}) def test_download_with_paths(self): """Test downloading a manifest while specifying the `paths` argument.""" @@ -329,11 +329,11 @@ def test_download_with_paths(self): # Check that dataset1 has been downloaded to the given directory. self.assertEqual(paths["dataset1"], temporary_directory) - self.assertEqual(os.listdir(paths["dataset1"]), ["file_1.txt", "file_0.txt", "sub-directory"]) + self.assertEqual(set(os.listdir(paths["dataset1"])), {"file_1.txt", "file_0.txt", "sub-directory"}) # Check that dataset2 has been downloaded to a temporary directory. self.assertNotEqual(paths["dataset2"], temporary_directory) - self.assertEqual(os.listdir(paths["dataset2"]), ["file_1.txt", "file_0.txt", "sub-directory"]) + self.assertEqual(set(os.listdir(paths["dataset2"])), {"file_1.txt", "file_0.txt", "sub-directory"}) def test_download_some_datasets_with_paths(self): """Test downloading a manifest while specifying the `paths` argument and setting `download_all=False`.""" @@ -350,7 +350,7 @@ def test_download_some_datasets_with_paths(self): # Check that dataset1 has been downloaded to the given directory and dataset2 hasn't been downloaded. self.assertEqual(paths, {"dataset1": temporary_directory}) - self.assertEqual(os.listdir(paths["dataset1"]), ["file_1.txt", "file_0.txt", "sub-directory"]) + self.assertEqual(set(os.listdir(paths["dataset1"])), {"file_1.txt", "file_0.txt", "sub-directory"}) self.assertEqual( logging_context.records[1].message, From afe5ffaaefebc78f82866ca932425f949a6fad22 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 21 Mar 2024 14:34:14 +0000 Subject: [PATCH 071/169] TST: Test `get_sruid_from_pub_sub_resource_name` --- tests/cloud/test_service_id.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/cloud/test_service_id.py b/tests/cloud/test_service_id.py index b820e35ac..5e637227d 100644 --- a/tests/cloud/test_service_id.py +++ b/tests/cloud/test_service_id.py @@ -10,6 +10,7 @@ from octue.cloud.service_id import ( convert_service_id_to_pub_sub_form, get_default_sruid, + get_sruid_from_pub_sub_resource_name, get_sruid_parts, raise_if_revision_not_registered, split_service_id, @@ -91,6 +92,13 @@ def test_convert_service_id_to_pub_sub_form(self): self.assertEqual(convert_service_id_to_pub_sub_form(service_id), pub_sub_service_id) +class TestGetSRUIDFromPubSubResourceName(unittest.TestCase): + def test_get_sruid_from_pub_sub_resource_name(self): + """Test that an SRUID can be extracted from a Pub/Sub resource name.""" + sruid = get_sruid_from_pub_sub_resource_name("octue.services.octue.example-service-cloud-run.0-3-2") + self.assertEqual(sruid, "octue/example-service-cloud-run:0.3.2") + + class TestValidateSRUID(unittest.TestCase): def test_error_raised_if_service_id_invalid(self): """Test that an error is raised if an invalid SRUID is given.""" From 452614801405fe59700a3de7d0ba83f6c661f5cb Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Fri, 22 Mar 2024 11:10:08 +0000 Subject: [PATCH 072/169] ENH: Add recipient and originator event attributes --- octue/cloud/emulators/_pub_sub.py | 1 + octue/cloud/pub_sub/events.py | 1 + octue/cloud/pub_sub/logging.py | 4 +++- octue/cloud/pub_sub/service.py | 33 +++++++++++++++++++------------ 4 files changed, 25 insertions(+), 14 deletions(-) diff --git a/octue/cloud/emulators/_pub_sub.py b/octue/cloud/emulators/_pub_sub.py index e497be5e5..5d2e724be 100644 --- a/octue/cloud/emulators/_pub_sub.py +++ b/octue/cloud/emulators/_pub_sub.py @@ -378,6 +378,7 @@ def ask( "save_diagnostics": save_diagnostics, "message_number": 0, "sender": service_id, + "originator": service_id, }, ) ) diff --git a/octue/cloud/pub_sub/events.py b/octue/cloud/pub_sub/events.py index cb515d5e5..1d7f8d855 100644 --- a/octue/cloud/pub_sub/events.py +++ b/octue/cloud/pub_sub/events.py @@ -37,6 +37,7 @@ def extract_event_and_attributes_from_pub_sub_message(message): "message_number": int(attributes["message_number"]), "version": attributes["version"], "sender": attributes.get("sender", "REMOTE"), # Backwards-compatible with previous event schema versions. + "originator": attributes["originator"], } if "forward_logs" in attributes: diff --git a/octue/cloud/pub_sub/logging.py b/octue/cloud/pub_sub/logging.py index 4df17b92e..a9e0c6303 100644 --- a/octue/cloud/pub_sub/logging.py +++ b/octue/cloud/pub_sub/logging.py @@ -15,10 +15,11 @@ class GoogleCloudPubSubHandler(logging.Handler): :return None: """ - def __init__(self, message_sender, topic, question_uuid, timeout=60, *args, **kwargs): + def __init__(self, message_sender, topic, question_uuid, originator, timeout=60, *args, **kwargs): super().__init__(*args, **kwargs) self.topic = topic self.question_uuid = question_uuid + self.originator = originator self.timeout = timeout self._send_message = message_sender @@ -38,6 +39,7 @@ def emit(self, record): attributes={ "question_uuid": self.question_uuid, "sender_type": "CHILD", # The sender type is repeated here as a string to avoid a circular import. + "originator": self.originator, }, ) diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index e7ed3c349..8f0add668 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -22,6 +22,7 @@ convert_service_id_to_pub_sub_form, create_sruid, get_default_sruid, + get_sruid_from_pub_sub_resource_name, raise_if_revision_not_registered, split_service_id, validate_sruid, @@ -204,6 +205,7 @@ def answer(self, question, heartbeat_interval=120, timeout=30): forward_logs, parent_sdk_version, save_diagnostics, + originator, ) = self._parse_question(question) except jsonschema.ValidationError: return @@ -212,12 +214,12 @@ def answer(self, question, heartbeat_interval=120, timeout=30): heartbeater = None try: - self._send_delivery_acknowledgment(topic, question_uuid) + self._send_delivery_acknowledgment(topic, question_uuid, originator) heartbeater = RepeatingTimer( interval=heartbeat_interval, function=self._send_heartbeat, - kwargs={"topic": topic, "question_uuid": question_uuid}, + kwargs={"topic": topic, "question_uuid": question_uuid, "originator": originator}, ) heartbeater.daemon = True @@ -228,6 +230,7 @@ def answer(self, question, heartbeat_interval=120, timeout=30): message_sender=self._send_message, topic=topic, question_uuid=question_uuid, + originator=originator, ) else: analysis_log_handler = None @@ -242,6 +245,7 @@ def answer(self, question, heartbeat_interval=120, timeout=30): self._send_monitor_message, topic=topic, question_uuid=question_uuid, + originator=originator, ), save_diagnostics=save_diagnostics, ) @@ -254,7 +258,7 @@ def answer(self, question, heartbeat_interval=120, timeout=30): self._send_message( message=result, topic=topic, - attributes={"question_uuid": question_uuid, "sender_type": CHILD_SENDER_TYPE}, + attributes={"question_uuid": question_uuid, "sender_type": CHILD_SENDER_TYPE, "originator": originator}, timeout=timeout, ) @@ -266,7 +270,7 @@ def answer(self, question, heartbeat_interval=120, timeout=30): heartbeater.cancel() warn_if_incompatible(child_sdk_version=self._local_sdk_version, parent_sdk_version=parent_sdk_version) - self.send_exception(topic, question_uuid, timeout=timeout) + self.send_exception(topic, question_uuid, originator, timeout=timeout) raise error def ask( @@ -398,7 +402,7 @@ def wait_for_answer( finally: subscription.delete() - def send_exception(self, topic, question_uuid, timeout=30): + def send_exception(self, topic, question_uuid, originator, timeout=30): """Serialise and send the exception being handled to the parent. :param octue.cloud.pub_sub.topic.Topic topic: @@ -417,7 +421,7 @@ def send_exception(self, topic, question_uuid, timeout=30): "exception_traceback": exception["traceback"], }, topic=topic, - attributes={"question_uuid": question_uuid, "sender_type": CHILD_SENDER_TYPE}, + attributes={"question_uuid": question_uuid, "sender_type": CHILD_SENDER_TYPE, "originator": originator}, timeout=timeout, ) @@ -437,6 +441,7 @@ def _send_message(self, message, topic, attributes=None, timeout=30): attributes["version"] = self._local_sdk_version attributes["message_number"] = topic.messages_published attributes["sender"] = self.id + attributes["recipient"] = get_sruid_from_pub_sub_resource_name(topic.name) converted_attributes = {} for key, value in attributes.items(): @@ -495,9 +500,10 @@ def _send_question( timeout=timeout, attributes={ "question_uuid": question_uuid, - "sender_type": PARENT_SENDER_TYPE, "forward_logs": forward_logs, "save_diagnostics": save_diagnostics, + "sender_type": PARENT_SENDER_TYPE, + "originator": self.id, }, ) @@ -505,7 +511,7 @@ def _send_question( future.result() logger.info("%r asked a question %r to service %r.", self, question_uuid, service_id) - def _send_delivery_acknowledgment(self, topic, question_uuid, timeout=30): + def _send_delivery_acknowledgment(self, topic, question_uuid, originator, timeout=30): """Send an acknowledgement of question receipt to the parent. :param octue.cloud.pub_sub.topic.Topic topic: topic to send the acknowledgement to @@ -520,12 +526,12 @@ def _send_delivery_acknowledgment(self, topic, question_uuid, timeout=30): }, topic=topic, timeout=timeout, - attributes={"question_uuid": question_uuid, "sender_type": CHILD_SENDER_TYPE}, + attributes={"question_uuid": question_uuid, "sender_type": CHILD_SENDER_TYPE, "originator": originator}, ) logger.info("%r acknowledged receipt of question %r.", self, question_uuid) - def _send_heartbeat(self, topic, question_uuid, timeout=30): + def _send_heartbeat(self, topic, question_uuid, originator, timeout=30): """Send a heartbeat to the parent, indicating that the service is alive. :param octue.cloud.pub_sub.topic.Topic topic: topic to send the heartbeat to @@ -540,12 +546,12 @@ def _send_heartbeat(self, topic, question_uuid, timeout=30): }, topic=topic, timeout=timeout, - attributes={"question_uuid": question_uuid, "sender_type": CHILD_SENDER_TYPE}, + attributes={"question_uuid": question_uuid, "sender_type": CHILD_SENDER_TYPE, "originator": originator}, ) logger.debug("Heartbeat sent by %r.", self) - def _send_monitor_message(self, data, topic, question_uuid, timeout=30): + def _send_monitor_message(self, data, topic, question_uuid, originator, timeout=30): """Send a monitor message to the parent. :param any data: the data to send as a monitor message @@ -558,7 +564,7 @@ def _send_monitor_message(self, data, topic, question_uuid, timeout=30): {"kind": "monitor_message", "data": data}, topic=topic, timeout=timeout, - attributes={"question_uuid": question_uuid, "sender_type": CHILD_SENDER_TYPE}, + attributes={"question_uuid": question_uuid, "sender_type": CHILD_SENDER_TYPE, "originator": originator}, ) logger.debug("Monitor message sent by %r.", self) @@ -594,4 +600,5 @@ def _parse_question(self, question): attributes["forward_logs"], attributes["version"], attributes["save_diagnostics"], + attributes["originator"], ) From 47c8af2a0ac4ad177a010a2a24cef61327f50653 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 27 Mar 2024 14:34:39 +0000 Subject: [PATCH 073/169] FIX: Require all question event attributes --- octue/cloud/pub_sub/events.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/octue/cloud/pub_sub/events.py b/octue/cloud/pub_sub/events.py index 1d7f8d855..646db4f05 100644 --- a/octue/cloud/pub_sub/events.py +++ b/octue/cloud/pub_sub/events.py @@ -31,6 +31,7 @@ def extract_event_and_attributes_from_pub_sub_message(message): # Cast attributes to a dictionary to avoid defaultdict-like behaviour from Pub/Sub message attributes container. attributes = dict(getattr_or_subscribe(message, "attributes")) + # Required for all events. converted_attributes = { "sender_type": attributes["sender_type"], "question_uuid": attributes["question_uuid"], @@ -40,11 +41,14 @@ def extract_event_and_attributes_from_pub_sub_message(message): "originator": attributes["originator"], } - if "forward_logs" in attributes: - converted_attributes["forward_logs"] = bool(int(attributes["forward_logs"])) - - if "save_diagnostics" in attributes: - converted_attributes["save_diagnostics"] = attributes["save_diagnostics"] + # Required for question events. + if attributes["sender_type"] == "PARENT": + converted_attributes.update( + { + "forward_logs": bool(int(attributes["forward_logs"])), + "save_diagnostics": attributes["save_diagnostics"], + } + ) try: # Parse event directly from Pub/Sub or Dataflow. From 55eee943988e2fb7d2f124113360fb10e1704271 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 27 Mar 2024 14:42:31 +0000 Subject: [PATCH 074/169] ENH: Give every event a UUID --- octue/cloud/pub_sub/service.py | 1 + 1 file changed, 1 insertion(+) diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index 8f0add668..ad1c3cd89 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -442,6 +442,7 @@ def _send_message(self, message, topic, attributes=None, timeout=30): attributes["message_number"] = topic.messages_published attributes["sender"] = self.id attributes["recipient"] = get_sruid_from_pub_sub_resource_name(topic.name) + attributes["uuid"] = str(uuid.uuid4()) converted_attributes = {} for key, value in attributes.items(): From f2cb0e2808b67e63ffa2de4fed773187b513f42a Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 27 Mar 2024 15:15:36 +0000 Subject: [PATCH 075/169] REF: Rename `message_number` to `ordering_key` BREAKING CHANGE: Update all services in your services network to use this version of `octue` or later. --- octue/cloud/emulators/_pub_sub.py | 2 +- octue/cloud/events/handler.py | 10 +++++----- octue/cloud/events/replayer.py | 2 +- octue/cloud/pub_sub/bigquery.py | 4 ++-- octue/cloud/pub_sub/events.py | 2 +- octue/cloud/pub_sub/service.py | 4 ++-- tests/cloud/pub_sub/test_events.py | 4 ++-- tests/cloud/pub_sub/test_service.py | 2 +- tests/data/events.json | 16 ++++++++-------- 9 files changed, 23 insertions(+), 23 deletions(-) diff --git a/octue/cloud/emulators/_pub_sub.py b/octue/cloud/emulators/_pub_sub.py index 5d2e724be..4479710ba 100644 --- a/octue/cloud/emulators/_pub_sub.py +++ b/octue/cloud/emulators/_pub_sub.py @@ -376,7 +376,7 @@ def ask( "forward_logs": subscribe_to_logs, "version": parent_sdk_version, "save_diagnostics": save_diagnostics, - "message_number": 0, + "ordering_key": 0, "sender": service_id, "originator": service_id, }, diff --git a/octue/cloud/events/handler.py b/octue/cloud/events/handler.py index c54d44b6b..28db74fd1 100644 --- a/octue/cloud/events/handler.py +++ b/octue/cloud/events/handler.py @@ -138,17 +138,17 @@ def _extract_and_enqueue_event(self, container): self.child_sruid = attributes.get("sender", "REMOTE") logger.debug("%r received an event related to question %r.", self.receiving_service, self.question_uuid) - event_number = attributes["message_number"] + ordering_key = attributes["ordering_key"] - if event_number in self.waiting_events: + if ordering_key in self.waiting_events: logger.warning( - "%r: Event with duplicate event number %d received for question %s - overwriting original event.", + "%r: Event with duplicate ordering key %d received for question %s - overwriting original event.", self.receiving_service, - event_number, + ordering_key, self.question_uuid, ) - self.waiting_events[event_number] = event + self.waiting_events[ordering_key] = event def _attempt_to_handle_waiting_events(self): """Attempt to handle events waiting in `self.waiting_events`. If these events aren't consecutive to the diff --git a/octue/cloud/events/replayer.py b/octue/cloud/events/replayer.py index f9b055119..efd2c5743 100644 --- a/octue/cloud/events/replayer.py +++ b/octue/cloud/events/replayer.py @@ -60,5 +60,5 @@ def _extract_event_and_attributes(self, container): :param dict container: the container of the event :return (any, dict): the event and its attributes """ - container["attributes"]["message_number"] = int(container["attributes"]["message_number"]) + container["attributes"]["ordering_key"] = int(container["attributes"]["ordering_key"]) return container["event"], container["attributes"] diff --git a/octue/cloud/pub_sub/bigquery.py b/octue/cloud/pub_sub/bigquery.py index 242037e5f..f5be5e1d0 100644 --- a/octue/cloud/pub_sub/bigquery.py +++ b/octue/cloud/pub_sub/bigquery.py @@ -58,7 +58,7 @@ def get_events(table_id, question_uuid, kind=None, limit=1000, include_pub_sub_m if isinstance(messages.at[0, "attributes"], str): messages["attributes"] = messages["attributes"].map(json.loads) - # Order messages by the message number. - messages = messages.iloc[messages["attributes"].str.get("message_number").astype(str).argsort()] + # Order messages by the ordering key. + messages = messages.iloc[messages["attributes"].str.get("ordering_key").astype(str).argsort()] messages.rename(columns={"data": "event"}, inplace=True) return messages.to_dict(orient="records") diff --git a/octue/cloud/pub_sub/events.py b/octue/cloud/pub_sub/events.py index 646db4f05..3f034c94e 100644 --- a/octue/cloud/pub_sub/events.py +++ b/octue/cloud/pub_sub/events.py @@ -35,7 +35,7 @@ def extract_event_and_attributes_from_pub_sub_message(message): converted_attributes = { "sender_type": attributes["sender_type"], "question_uuid": attributes["question_uuid"], - "message_number": int(attributes["message_number"]), + "ordering_key": int(attributes["ordering_key"]), "version": attributes["version"], "sender": attributes.get("sender", "REMOTE"), # Backwards-compatible with previous event schema versions. "originator": attributes["originator"], diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index ad1c3cd89..640a3f0fd 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -36,7 +36,7 @@ logger = logging.getLogger(__name__) -# A lock to ensure only one message can be sent at a time so that the message number is incremented correctly when +# A lock to ensure only one message can be sent at a time so that the ordering key is incremented correctly when # messages are being sent on multiple threads (e.g. via the main thread and a periodic monitor message thread). This # avoids 1) messages overwriting each other in the parent's message handler and 2) messages losing their order. send_message_lock = threading.Lock() @@ -439,7 +439,7 @@ def _send_message(self, message, topic, attributes=None, timeout=30): with send_message_lock: attributes["version"] = self._local_sdk_version - attributes["message_number"] = topic.messages_published + attributes["ordering_key"] = topic.messages_published attributes["sender"] = self.id attributes["recipient"] = get_sruid_from_pub_sub_resource_name(topic.name) attributes["uuid"] = str(uuid.uuid4()) diff --git a/tests/cloud/pub_sub/test_events.py b/tests/cloud/pub_sub/test_events.py index 2f97ba897..be3425c4c 100644 --- a/tests/cloud/pub_sub/test_events.py +++ b/tests/cloud/pub_sub/test_events.py @@ -488,7 +488,7 @@ def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): # Send another message. child._send_message( message={"kind": "test", "order": 5}, - attributes={"message_number": 5, "question_uuid": question_uuid, "sender_type": "CHILD"}, + attributes={"ordering_key": 5, "question_uuid": question_uuid, "sender_type": "CHILD"}, topic=mock_topic, ) @@ -597,7 +597,7 @@ def test_pull_and_enqueue_available_events(self): mock_message, attributes={ "sender_type": "CHILD", - "message_number": 0, + "ordering_key": 0, "question_uuid": question_uuid, "version": "0.50.0", }, diff --git a/tests/cloud/pub_sub/test_service.py b/tests/cloud/pub_sub/test_service.py index 37adc3599..1558e64af 100644 --- a/tests/cloud/pub_sub/test_service.py +++ b/tests/cloud/pub_sub/test_service.py @@ -326,7 +326,7 @@ def mock_app(analysis): with self.service_patcher: child.serve() subscription, _ = parent.ask(service_id=child.id, input_values={}, subscribe_to_logs=True) - parent.wait_for_answer(subscription) + parent.wait_for_answer(subscription, timeout=10) error_logged = False diff --git a/tests/data/events.json b/tests/data/events.json index d1e9e1b2a..ec42d1409 100644 --- a/tests/data/events.json +++ b/tests/data/events.json @@ -5,7 +5,7 @@ "kind": "delivery_acknowledgement" }, "attributes": { - "message_number": "0", + "ordering_key": "0", "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", "sender_type": "CHILD", "version": "0.51.0", @@ -39,7 +39,7 @@ } }, "attributes": { - "message_number": "1", + "ordering_key": "1", "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", "sender_type": "CHILD", "version": "0.51.0", @@ -73,7 +73,7 @@ } }, "attributes": { - "message_number": "2", + "ordering_key": "2", "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", "sender_type": "CHILD", "version": "0.51.0", @@ -107,7 +107,7 @@ } }, "attributes": { - "message_number": "3", + "ordering_key": "3", "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", "sender_type": "CHILD", "version": "0.51.0", @@ -141,7 +141,7 @@ } }, "attributes": { - "message_number": "4", + "ordering_key": "4", "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", "sender_type": "CHILD", "version": "0.51.0", @@ -175,7 +175,7 @@ } }, "attributes": { - "message_number": "5", + "ordering_key": "5", "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", "sender_type": "CHILD", "version": "0.51.0", @@ -204,7 +204,7 @@ "output_values": [1, 2, 3, 4, 5] }, "attributes": { - "message_number": "6", + "ordering_key": "6", "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", "sender_type": "CHILD", "version": "0.51.0", @@ -217,7 +217,7 @@ "kind": "heartbeat" }, "attributes": { - "message_number": "7", + "ordering_key": "7", "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", "sender_type": "CHILD", "version": "0.51.0", From ed64727deb4f4d8c9f4a60ac28385ec337a00930 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 27 Mar 2024 15:16:31 +0000 Subject: [PATCH 076/169] FIX: Only update earliest waiting message number if waiting messages --- octue/cloud/pub_sub/events.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/octue/cloud/pub_sub/events.py b/octue/cloud/pub_sub/events.py index 3f034c94e..f99fb9168 100644 --- a/octue/cloud/pub_sub/events.py +++ b/octue/cloud/pub_sub/events.py @@ -245,7 +245,8 @@ def _pull_and_enqueue_available_events(self, timeout): for event in pull_response.received_messages: self._extract_and_enqueue_event(event) - self._earliest_waiting_event_number = min(self.waiting_events.keys()) + if self.waiting_events: + self._earliest_waiting_event_number = min(self.waiting_events.keys()) def _extract_event_and_attributes(self, container): """Extract an event and its attributes from the Pub/Sub message. From 4daa88e773f4f39f80da823267111754dde59a0a Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 27 Mar 2024 16:30:18 +0000 Subject: [PATCH 077/169] ENH: Add mandatory originator, sender, and recipient event attributes BREAKING CHANGE: Use this or a newer version of `octue` for all services in your network. --- octue/cloud/emulators/_pub_sub.py | 18 +++++-- octue/cloud/events/handler.py | 10 ++-- octue/cloud/pub_sub/events.py | 13 +++-- octue/cloud/pub_sub/logging.py | 3 +- octue/cloud/pub_sub/service.py | 58 +++++++++++++++++----- tests/cloud/pub_sub/test_events.py | 79 ++++++++++++++++++++++++++---- 6 files changed, 148 insertions(+), 33 deletions(-) diff --git a/octue/cloud/emulators/_pub_sub.py b/octue/cloud/emulators/_pub_sub.py index 4479710ba..d86587c48 100644 --- a/octue/cloud/emulators/_pub_sub.py +++ b/octue/cloud/emulators/_pub_sub.py @@ -6,7 +6,7 @@ from octue.cloud.pub_sub import Subscription, Topic from octue.cloud.pub_sub.service import ANSWERS_NAMESPACE, PARENT_SENDER_TYPE, Service -from octue.cloud.service_id import convert_service_id_to_pub_sub_form +from octue.cloud.service_id import convert_service_id_to_pub_sub_form, split_service_id from octue.resources import Manifest from octue.utils.dictionaries import make_minimal_dictionary from octue.utils.encoders import OctueJSONEncoder @@ -366,19 +366,29 @@ def ask( if input_manifest is not None: question["input_manifest"] = input_manifest.to_primitive() + originator_namespace, originator_name, originator_revision_tag = split_service_id(self.id) + recipient_namespace, recipient_name, recipient_revision_tag = split_service_id(service_id) + try: self.children[service_id].answer( MockMessage.from_primitive( data=question, attributes={ - "sender_type": PARENT_SENDER_TYPE, "question_uuid": question_uuid, "forward_logs": subscribe_to_logs, "version": parent_sdk_version, "save_diagnostics": save_diagnostics, "ordering_key": 0, - "sender": service_id, - "originator": service_id, + "originator_namespace": originator_namespace, + "originator_name": originator_name, + "originator_revision_tag": originator_revision_tag, + "sender_namespace": originator_namespace, + "sender_name": originator_name, + "sender_revision_tag": originator_revision_tag, + "sender_type": PARENT_SENDER_TYPE, + "recipient_namespace": recipient_namespace, + "recipient_name": recipient_name, + "recipient_revision_tag": recipient_revision_tag, }, ) ) diff --git a/octue/cloud/events/handler.py b/octue/cloud/events/handler.py index 28db74fd1..515e25631 100644 --- a/octue/cloud/events/handler.py +++ b/octue/cloud/events/handler.py @@ -9,6 +9,7 @@ from octue.cloud import EXCEPTIONS_MAPPING from octue.cloud.events.validation import SERVICE_COMMUNICATION_SCHEMA, is_event_valid +from octue.cloud.service_id import create_sruid from octue.definitions import GOOGLE_COMPUTE_PROVIDERS from octue.log_handlers import COLOUR_PALETTE from octue.resources.manifest import Manifest @@ -131,12 +132,15 @@ def _extract_and_enqueue_event(self, container): # Get the child's SRUID and Octue SDK version from the first event. if not self._child_sdk_version: + self.child_sruid = create_sruid( + namespace=attributes["sender_namespace"], + name=attributes["sender_name"], + revision_tag=attributes["sender_revision_tag"], + ) + self.question_uuid = attributes.get("question_uuid") self._child_sdk_version = attributes["version"] - # Backwards-compatible with previous event schema versions. - self.child_sruid = attributes.get("sender", "REMOTE") - logger.debug("%r received an event related to question %r.", self.receiving_service, self.question_uuid) ordering_key = attributes["ordering_key"] diff --git a/octue/cloud/pub_sub/events.py b/octue/cloud/pub_sub/events.py index f99fb9168..73a746ca2 100644 --- a/octue/cloud/pub_sub/events.py +++ b/octue/cloud/pub_sub/events.py @@ -33,12 +33,19 @@ def extract_event_and_attributes_from_pub_sub_message(message): # Required for all events. converted_attributes = { - "sender_type": attributes["sender_type"], "question_uuid": attributes["question_uuid"], "ordering_key": int(attributes["ordering_key"]), "version": attributes["version"], - "sender": attributes.get("sender", "REMOTE"), # Backwards-compatible with previous event schema versions. - "originator": attributes["originator"], + "originator_namespace": attributes["originator_namespace"], + "originator_name": attributes["originator_name"], + "originator_revision_tag": attributes["originator_revision_tag"], + "sender_namespace": attributes["sender_namespace"], + "sender_name": attributes["sender_name"], + "sender_revision_tag": attributes["sender_revision_tag"], + "sender_type": attributes["sender_type"], + "recipient_namespace": attributes["recipient_namespace"], + "recipient_name": attributes["recipient_name"], + "recipient_revision_tag": attributes["recipient_revision_tag"], } # Required for question events. diff --git a/octue/cloud/pub_sub/logging.py b/octue/cloud/pub_sub/logging.py index a9e0c6303..902e8773c 100644 --- a/octue/cloud/pub_sub/logging.py +++ b/octue/cloud/pub_sub/logging.py @@ -11,6 +11,7 @@ class GoogleCloudPubSubHandler(logging.Handler): :param callable message_sender: the `_send_message` method of the service that instantiated this instance :param octue.cloud.pub_sub.topic.Topic topic: topic to publish log records to :param str question_uuid: the UUID of the question to handle log records for + :param str originator: the SRUID of the service that asked the question these log records are related to :param float timeout: timeout in seconds for attempting to publish each log record :return None: """ @@ -36,10 +37,10 @@ def emit(self, record): "log_record": self._convert_log_record_to_primitives(record), }, topic=self.topic, + originator=self.originator, attributes={ "question_uuid": self.question_uuid, "sender_type": "CHILD", # The sender type is repeated here as a string to avoid a circular import. - "originator": self.originator, }, ) diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index 640a3f0fd..7b4a8794a 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -258,7 +258,8 @@ def answer(self, question, heartbeat_interval=120, timeout=30): self._send_message( message=result, topic=topic, - attributes={"question_uuid": question_uuid, "sender_type": CHILD_SENDER_TYPE, "originator": originator}, + originator=originator, + attributes={"question_uuid": question_uuid, "sender_type": CHILD_SENDER_TYPE}, timeout=timeout, ) @@ -407,6 +408,7 @@ def send_exception(self, topic, question_uuid, originator, timeout=30): :param octue.cloud.pub_sub.topic.Topic topic: :param str question_uuid: + :param str originator: the SRUID of the service that asked the question this event is related to :param float|None timeout: time in seconds to keep retrying sending of the exception :return None: """ @@ -421,28 +423,48 @@ def send_exception(self, topic, question_uuid, originator, timeout=30): "exception_traceback": exception["traceback"], }, topic=topic, - attributes={"question_uuid": question_uuid, "sender_type": CHILD_SENDER_TYPE, "originator": originator}, + originator=originator, + attributes={"question_uuid": question_uuid, "sender_type": CHILD_SENDER_TYPE}, timeout=timeout, ) - def _send_message(self, message, topic, attributes=None, timeout=30): + def _send_message(self, message, topic, originator, attributes=None, timeout=30): """Send a JSON-serialised message to the given topic with optional message attributes and increment the `messages_published` attribute of the topic by one. This method is thread-safe. :param dict message: JSON-serialisable data to send as a message :param octue.cloud.pub_sub.topic.Topic topic: the Pub/Sub topic to send the message to + :param str originator: the SRUID of the service that asked the question this event is related to :param dict|None attributes: key-value pairs to attach to the message - the values must be strings or bytes :param int|float timeout: the timeout for sending the message in seconds :return google.cloud.pubsub_v1.publisher.futures.Future: """ attributes = attributes or {} + originator_namespace, originator_name, originator_revision_tag = split_service_id(originator) + sender_namespace, sender_name, sender_revision_tag = split_service_id(self.id) + + recipient_namespace, recipient_name, recipient_revision_tag = split_service_id( + get_sruid_from_pub_sub_resource_name(topic.name) + ) + + attributes["originator_namespace"] = originator_namespace + attributes["originator_name"] = originator_name + attributes["originator_revision_tag"] = originator_revision_tag + + attributes["sender_namespace"] = sender_namespace + attributes["sender_name"] = sender_name + attributes["sender_revision_tag"] = sender_revision_tag + + attributes["recipient_namespace"] = recipient_namespace + attributes["recipient_name"] = recipient_name + attributes["recipient_revision_tag"] = recipient_revision_tag + with send_message_lock: attributes["version"] = self._local_sdk_version attributes["ordering_key"] = topic.messages_published - attributes["sender"] = self.id - attributes["recipient"] = get_sruid_from_pub_sub_resource_name(topic.name) attributes["uuid"] = str(uuid.uuid4()) + converted_attributes = {} for key, value in attributes.items(): @@ -499,12 +521,12 @@ def _send_question( message=question, topic=topic, timeout=timeout, + originator=self.id, attributes={ "question_uuid": question_uuid, "forward_logs": forward_logs, "save_diagnostics": save_diagnostics, "sender_type": PARENT_SENDER_TYPE, - "originator": self.id, }, ) @@ -517,6 +539,7 @@ def _send_delivery_acknowledgment(self, topic, question_uuid, originator, timeou :param octue.cloud.pub_sub.topic.Topic topic: topic to send the acknowledgement to :param str question_uuid: + :param str originator: the SRUID of the service that asked the question this event is related to :param float timeout: time in seconds after which to give up sending :return None: """ @@ -527,7 +550,8 @@ def _send_delivery_acknowledgment(self, topic, question_uuid, originator, timeou }, topic=topic, timeout=timeout, - attributes={"question_uuid": question_uuid, "sender_type": CHILD_SENDER_TYPE, "originator": originator}, + originator=originator, + attributes={"question_uuid": question_uuid, "sender_type": CHILD_SENDER_TYPE}, ) logger.info("%r acknowledged receipt of question %r.", self, question_uuid) @@ -537,6 +561,7 @@ def _send_heartbeat(self, topic, question_uuid, originator, timeout=30): :param octue.cloud.pub_sub.topic.Topic topic: topic to send the heartbeat to :param str question_uuid: + :param str originator: the SRUID of the service that asked the question this event is related to :param float timeout: time in seconds after which to give up sending :return None: """ @@ -546,8 +571,9 @@ def _send_heartbeat(self, topic, question_uuid, originator, timeout=30): "datetime": datetime.datetime.utcnow().isoformat(), }, topic=topic, + originator=originator, timeout=timeout, - attributes={"question_uuid": question_uuid, "sender_type": CHILD_SENDER_TYPE, "originator": originator}, + attributes={"question_uuid": question_uuid, "sender_type": CHILD_SENDER_TYPE}, ) logger.debug("Heartbeat sent by %r.", self) @@ -558,14 +584,16 @@ def _send_monitor_message(self, data, topic, question_uuid, originator, timeout= :param any data: the data to send as a monitor message :param octue.cloud.pub_sub.topic.Topic topic: the topic to send the message to :param str question_uuid: + :param str originator: the SRUID of the service that asked the question this event is related to :param float timeout: time in seconds to retry sending the message :return None: """ self._send_message( {"kind": "monitor_message", "data": data}, topic=topic, + originator=originator, timeout=timeout, - attributes={"question_uuid": question_uuid, "sender_type": CHILD_SENDER_TYPE, "originator": originator}, + attributes={"question_uuid": question_uuid, "sender_type": CHILD_SENDER_TYPE}, ) logger.debug("Monitor message sent by %r.", self) @@ -574,11 +602,11 @@ def _parse_question(self, question): """Parse a question in the Google Cloud Run or Google Pub/Sub format. :param dict|google.cloud.pubsub_v1.subscriber.message.Message question: the question to parse in Google Cloud Run or Google Pub/Sub format - :return (dict, str, bool, str, str): the question's event and its attributes (question UUID, whether to forward logs, the Octue SDK version of the parent, and whether to save diagnostics) + :return (dict, str, bool, str, str, str): the question's event and its attributes (question UUID, whether to forward logs, the Octue SDK version of the parent, whether to save diagnostics, and the SRUID of the service revision that asked the question) """ logger.info("%r received a question.", self) - # Acknowledge it if it's directly from Pub/Sub + # Acknowledge the question if it's directly from Pub/Sub. if hasattr(question, "ack"): question.ack() @@ -595,11 +623,17 @@ def _parse_question(self, question): logger.info("%r parsed the question successfully.", self) + originator = create_sruid( + namespace=attributes["originator_namespace"], + name=attributes["originator_name"], + revision_tag=attributes["originator_revision_tag"], + ) + return ( event, attributes["question_uuid"], attributes["forward_logs"], attributes["version"], attributes["save_diagnostics"], - attributes["originator"], + originator, ) diff --git a/tests/cloud/pub_sub/test_events.py b/tests/cloud/pub_sub/test_events.py index be3425c4c..811ee8e56 100644 --- a/tests/cloud/pub_sub/test_events.py +++ b/tests/cloud/pub_sub/test_events.py @@ -13,6 +13,7 @@ ) from octue.cloud.emulators.child import ServicePatcher from octue.cloud.pub_sub.events import GoogleCloudPubSubEventHandler +from octue.cloud.service_id import split_service_id from octue.resources.service_backends import GCPPubSubBackend from tests import TEST_PROJECT_NAME from tests.base import BaseTestCase @@ -72,7 +73,12 @@ def test_in_order_messages_are_handled_in_order(self): ] for message in messages: - child._send_message(message=message["event"], attributes=message["attributes"], topic=mock_topic) + child._send_message( + message=message["event"], + attributes=message["attributes"], + topic=mock_topic, + originator=parent.id, + ) result = event_handler.handle_events() self.assertEqual(result, "This is the result.") @@ -117,7 +123,12 @@ def test_out_of_order_messages_are_handled_in_order(self): for message in messages: mock_topic.messages_published = message["event"]["order"] - child._send_message(message=message["event"], attributes=message["attributes"], topic=mock_topic) + child._send_message( + message=message["event"], + attributes=message["attributes"], + topic=mock_topic, + originator=parent.id, + ) result = event_handler.handle_events() @@ -170,7 +181,12 @@ def test_out_of_order_messages_with_end_message_first_are_handled_in_order(self) for message in messages: mock_topic.messages_published = message["event"]["order"] - child._send_message(message=message["event"], attributes=message["attributes"], topic=mock_topic) + child._send_message( + message=message["event"], + attributes=message["attributes"], + topic=mock_topic, + originator=parent.id, + ) result = event_handler.handle_events() @@ -216,7 +232,12 @@ def test_no_timeout(self): ] for message in messages: - child._send_message(message=message["event"], attributes=message["attributes"], topic=mock_topic) + child._send_message( + message=message["event"], + attributes=message["attributes"], + topic=mock_topic, + originator=parent.id, + ) result = event_handler.handle_events(timeout=None) @@ -247,7 +268,12 @@ def test_delivery_acknowledgement(self): ] for message in messages: - child._send_message(message=message["event"], attributes=message["attributes"], topic=mock_topic) + child._send_message( + message=message["event"], + attributes=message["attributes"], + topic=mock_topic, + originator=parent.id, + ) result = event_handler.handle_events() self.assertEqual(result, {"output_values": None, "output_manifest": None}) @@ -305,7 +331,12 @@ def test_error_not_raised_if_heartbeat_has_been_received_in_maximum_allowed_inte ] for message in messages: - child._send_message(message=message["event"], attributes=message["attributes"], topic=mock_topic) + child._send_message( + message=message["event"], + attributes=message["attributes"], + topic=mock_topic, + originator=parent.id, + ) with patch( "octue.cloud.pub_sub.events.GoogleCloudPubSubEventHandler._time_since_last_heartbeat", @@ -376,7 +407,12 @@ def test_missing_messages_at_start_can_be_skipped(self): ] for message in messages: - child._send_message(message=message["event"], attributes=message["attributes"], topic=mock_topic) + child._send_message( + message=message["event"], + attributes=message["attributes"], + topic=mock_topic, + originator=parent.id, + ) result = event_handler.handle_events() @@ -423,7 +459,12 @@ def test_missing_messages_in_middle_can_skipped(self): ] for message in messages: - child._send_message(message=message["event"], attributes=message["attributes"], topic=mock_topic) + child._send_message( + message=message["event"], + attributes=message["attributes"], + topic=mock_topic, + originator=parent.id, + ) # Simulate missing messages. mock_topic.messages_published = 5 @@ -433,6 +474,7 @@ def test_missing_messages_in_middle_can_skipped(self): message={"kind": "finish-test", "order": 5}, attributes={"question_uuid": question_uuid, "sender_type": "CHILD"}, topic=mock_topic, + originator=parent.id, ) event_handler.handle_events() @@ -480,7 +522,9 @@ def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): ] for message in messages: - child._send_message(message=message["event"], attributes=message["attributes"], topic=mock_topic) + child._send_message( + message=message["event"], attributes=message["attributes"], topic=mock_topic, originator=parent.id + ) # Simulate missing messages. mock_topic.messages_published = 5 @@ -490,6 +534,7 @@ def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): message={"kind": "test", "order": 5}, attributes={"ordering_key": 5, "question_uuid": question_uuid, "sender_type": "CHILD"}, topic=mock_topic, + originator=parent.id, ) # Simulate more missing messages. @@ -516,7 +561,9 @@ def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): ] for message in messages: - child._send_message(message=message["event"], attributes=message["attributes"], topic=mock_topic) + child._send_message( + message=message["event"], attributes=message["attributes"], topic=mock_topic, originator=parent.id + ) event_handler.handle_events() @@ -558,6 +605,7 @@ def test_all_messages_missing_apart_from_result(self): message={"kind": "finish-test", "order": 1000}, attributes={"question_uuid": question_uuid, "sender_type": "CHILD"}, topic=mock_topic, + originator=parent.id, ) event_handler.handle_events() @@ -592,6 +640,8 @@ def test_pull_and_enqueue_available_events(self): # Enqueue a mock message for a mock subscription to receive. mock_message = {"kind": "test"} + originator_namespace, originator_name, originator_revision_tag = split_service_id(parent.id) + SUBSCRIPTIONS[mock_subscription.name] = [ MockMessage.from_primitive( mock_message, @@ -600,6 +650,15 @@ def test_pull_and_enqueue_available_events(self): "ordering_key": 0, "question_uuid": question_uuid, "version": "0.50.0", + "originator_namespace": originator_namespace, + "originator_name": originator_name, + "originator_revision_tag": originator_revision_tag, + "sender_namespace": originator_namespace, + "sender_name": originator_name, + "sender_revision_tag": originator_revision_tag, + "recipient_namespace": "my-org", + "recipient_name": "my-service", + "recipient_revision_tag": "1.0.0", }, ) ] From 78117e2904de7d5124ddb17361afc13342c7e349 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 27 Mar 2024 17:14:35 +0000 Subject: [PATCH 078/169] FIX: Rename `ordering_key` to `order` This avoids the pub/sub publisher assuming we're using the Google Pub/Sub ordering key and not our own. skipci --- octue/cloud/emulators/_pub_sub.py | 2 +- octue/cloud/events/handler.py | 10 +++++----- octue/cloud/events/replayer.py | 2 +- octue/cloud/pub_sub/bigquery.py | 4 ++-- octue/cloud/pub_sub/events.py | 2 +- octue/cloud/pub_sub/service.py | 8 ++++---- tests/cloud/pub_sub/test_events.py | 4 ++-- tests/cloud/pub_sub/test_logging.py | 8 +++++--- 8 files changed, 21 insertions(+), 19 deletions(-) diff --git a/octue/cloud/emulators/_pub_sub.py b/octue/cloud/emulators/_pub_sub.py index d86587c48..353932d20 100644 --- a/octue/cloud/emulators/_pub_sub.py +++ b/octue/cloud/emulators/_pub_sub.py @@ -378,7 +378,7 @@ def ask( "forward_logs": subscribe_to_logs, "version": parent_sdk_version, "save_diagnostics": save_diagnostics, - "ordering_key": 0, + "order": 0, "originator_namespace": originator_namespace, "originator_name": originator_name, "originator_revision_tag": originator_revision_tag, diff --git a/octue/cloud/events/handler.py b/octue/cloud/events/handler.py index 515e25631..f71d74c33 100644 --- a/octue/cloud/events/handler.py +++ b/octue/cloud/events/handler.py @@ -142,17 +142,17 @@ def _extract_and_enqueue_event(self, container): self._child_sdk_version = attributes["version"] logger.debug("%r received an event related to question %r.", self.receiving_service, self.question_uuid) - ordering_key = attributes["ordering_key"] + order = attributes["order"] - if ordering_key in self.waiting_events: + if order in self.waiting_events: logger.warning( - "%r: Event with duplicate ordering key %d received for question %s - overwriting original event.", + "%r: Event with duplicate order %d received for question %s - overwriting original event.", self.receiving_service, - ordering_key, + order, self.question_uuid, ) - self.waiting_events[ordering_key] = event + self.waiting_events[order] = event def _attempt_to_handle_waiting_events(self): """Attempt to handle events waiting in `self.waiting_events`. If these events aren't consecutive to the diff --git a/octue/cloud/events/replayer.py b/octue/cloud/events/replayer.py index efd2c5743..13e3d2a5c 100644 --- a/octue/cloud/events/replayer.py +++ b/octue/cloud/events/replayer.py @@ -60,5 +60,5 @@ def _extract_event_and_attributes(self, container): :param dict container: the container of the event :return (any, dict): the event and its attributes """ - container["attributes"]["ordering_key"] = int(container["attributes"]["ordering_key"]) + container["attributes"]["order"] = int(container["attributes"]["order"]) return container["event"], container["attributes"] diff --git a/octue/cloud/pub_sub/bigquery.py b/octue/cloud/pub_sub/bigquery.py index f5be5e1d0..409372039 100644 --- a/octue/cloud/pub_sub/bigquery.py +++ b/octue/cloud/pub_sub/bigquery.py @@ -58,7 +58,7 @@ def get_events(table_id, question_uuid, kind=None, limit=1000, include_pub_sub_m if isinstance(messages.at[0, "attributes"], str): messages["attributes"] = messages["attributes"].map(json.loads) - # Order messages by the ordering key. - messages = messages.iloc[messages["attributes"].str.get("ordering_key").astype(str).argsort()] + # Order messages. + messages = messages.iloc[messages["attributes"].str.get("order").astype(str).argsort()] messages.rename(columns={"data": "event"}, inplace=True) return messages.to_dict(orient="records") diff --git a/octue/cloud/pub_sub/events.py b/octue/cloud/pub_sub/events.py index 73a746ca2..587f656c0 100644 --- a/octue/cloud/pub_sub/events.py +++ b/octue/cloud/pub_sub/events.py @@ -34,7 +34,7 @@ def extract_event_and_attributes_from_pub_sub_message(message): # Required for all events. converted_attributes = { "question_uuid": attributes["question_uuid"], - "ordering_key": int(attributes["ordering_key"]), + "order": int(attributes["order"]), "version": attributes["version"], "originator_namespace": attributes["originator_namespace"], "originator_name": attributes["originator_name"], diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index 7b4a8794a..d3c095b74 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -36,9 +36,9 @@ logger = logging.getLogger(__name__) -# A lock to ensure only one message can be sent at a time so that the ordering key is incremented correctly when -# messages are being sent on multiple threads (e.g. via the main thread and a periodic monitor message thread). This -# avoids 1) messages overwriting each other in the parent's message handler and 2) messages losing their order. +# A lock to ensure only one message can be sent at a time so that the order is incremented correctly when messages are +# being sent on multiple threads (e.g. via the main thread and a periodic monitor message thread). This avoids 1) +# messages overwriting each other in the parent's message handler and 2) messages losing their order. send_message_lock = threading.Lock() DEFAULT_NAMESPACE = "default" @@ -462,7 +462,7 @@ def _send_message(self, message, topic, originator, attributes=None, timeout=30) with send_message_lock: attributes["version"] = self._local_sdk_version - attributes["ordering_key"] = topic.messages_published + attributes["order"] = topic.messages_published attributes["uuid"] = str(uuid.uuid4()) converted_attributes = {} diff --git a/tests/cloud/pub_sub/test_events.py b/tests/cloud/pub_sub/test_events.py index 811ee8e56..2d2ed3359 100644 --- a/tests/cloud/pub_sub/test_events.py +++ b/tests/cloud/pub_sub/test_events.py @@ -532,7 +532,7 @@ def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): # Send another message. child._send_message( message={"kind": "test", "order": 5}, - attributes={"ordering_key": 5, "question_uuid": question_uuid, "sender_type": "CHILD"}, + attributes={"order": 5, "question_uuid": question_uuid, "sender_type": "CHILD"}, topic=mock_topic, originator=parent.id, ) @@ -647,7 +647,7 @@ def test_pull_and_enqueue_available_events(self): mock_message, attributes={ "sender_type": "CHILD", - "ordering_key": 0, + "order": 0, "question_uuid": question_uuid, "version": "0.50.0", "originator_namespace": originator_namespace, diff --git a/tests/cloud/pub_sub/test_logging.py b/tests/cloud/pub_sub/test_logging.py index 5b333cb3c..fee4a5f01 100644 --- a/tests/cloud/pub_sub/test_logging.py +++ b/tests/cloud/pub_sub/test_logging.py @@ -17,11 +17,11 @@ def __repr__(self): class TestGoogleCloudPubSubHandler(BaseTestCase): def test_emit(self): """Test the log message is published when `GoogleCloudPubSubHandler.emit` is called.""" - topic = MockTopic(name="world", project_name="blah") + topic = MockTopic(name="octue.my-service.3-3-3", project_name="blah") topic.create() question_uuid = "96d69278-44ac-4631-aeea-c90fb08a1b2b" - subscription = MockSubscription(name=f"world.answers.{question_uuid}", topic=topic) + subscription = MockSubscription(name=f"octue.my-service.3-3-3.answers.{question_uuid}", topic=topic) subscription.create() log_record = makeLogRecord({"msg": "Starting analysis."}) @@ -33,6 +33,7 @@ def test_emit(self): message_sender=service._send_message, topic=topic, question_uuid=question_uuid, + originator=service.id, ).emit(log_record) self.assertEqual( @@ -44,7 +45,7 @@ def test_emit_with_non_json_serialisable_args(self): """Test that non-JSON-serialisable arguments to log messages are converted to their string representation before being serialised and published to the Pub/Sub topic. """ - topic = MockTopic(name="world-1", project_name="blah") + topic = MockTopic(name="octue.my-service.3-3-4", project_name="blah") topic.create() non_json_serialisable_thing = NonJSONSerialisable() @@ -65,6 +66,7 @@ def test_emit_with_non_json_serialisable_args(self): message_sender=service._send_message, topic=topic, question_uuid="question-uuid", + originator=service.id, ).emit(record) self.assertEqual( From 17e6dfad23695cdc89a11091d7fe268ec40cc03d Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 28 Mar 2024 15:11:27 +0000 Subject: [PATCH 079/169] REF: Rename `version` attribute to `sender_sdk_version` BREAKING CHANGE: Update `octue` on all services in your network to this version or higher --- octue/cloud/emulators/_pub_sub.py | 2 +- octue/cloud/events/handler.py | 4 ++-- octue/cloud/pub_sub/events.py | 2 +- octue/cloud/pub_sub/service.py | 6 +++--- tests/cloud/pub_sub/test_events.py | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/octue/cloud/emulators/_pub_sub.py b/octue/cloud/emulators/_pub_sub.py index 353932d20..dbd1b40bd 100644 --- a/octue/cloud/emulators/_pub_sub.py +++ b/octue/cloud/emulators/_pub_sub.py @@ -376,7 +376,7 @@ def ask( attributes={ "question_uuid": question_uuid, "forward_logs": subscribe_to_logs, - "version": parent_sdk_version, + "sender_sdk_version": parent_sdk_version, "save_diagnostics": save_diagnostics, "order": 0, "originator_namespace": originator_namespace, diff --git a/octue/cloud/events/handler.py b/octue/cloud/events/handler.py index f71d74c33..292433cae 100644 --- a/octue/cloud/events/handler.py +++ b/octue/cloud/events/handler.py @@ -125,7 +125,7 @@ def _extract_and_enqueue_event(self, container): attributes=attributes, receiving_service=self.receiving_service, parent_sdk_version=PARENT_SDK_VERSION, - child_sdk_version=attributes.get("version"), + child_sdk_version=attributes["sender_sdk_version"], schema=self.schema, ): return @@ -139,7 +139,7 @@ def _extract_and_enqueue_event(self, container): ) self.question_uuid = attributes.get("question_uuid") - self._child_sdk_version = attributes["version"] + self._child_sdk_version = attributes["sender_sdk_version"] logger.debug("%r received an event related to question %r.", self.receiving_service, self.question_uuid) order = attributes["order"] diff --git a/octue/cloud/pub_sub/events.py b/octue/cloud/pub_sub/events.py index 587f656c0..8423009b1 100644 --- a/octue/cloud/pub_sub/events.py +++ b/octue/cloud/pub_sub/events.py @@ -35,7 +35,7 @@ def extract_event_and_attributes_from_pub_sub_message(message): converted_attributes = { "question_uuid": attributes["question_uuid"], "order": int(attributes["order"]), - "version": attributes["version"], + "sender_sdk_version": attributes["sender_sdk_version"], "originator_namespace": attributes["originator_namespace"], "originator_name": attributes["originator_name"], "originator_revision_tag": attributes["originator_revision_tag"], diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index d3c095b74..54dc599ff 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -461,7 +461,7 @@ def _send_message(self, message, topic, originator, attributes=None, timeout=30) attributes["recipient_revision_tag"] = recipient_revision_tag with send_message_lock: - attributes["version"] = self._local_sdk_version + attributes["sender_sdk_version"] = self._local_sdk_version attributes["order"] = topic.messages_published attributes["uuid"] = str(uuid.uuid4()) @@ -617,7 +617,7 @@ def _parse_question(self, question): event=event_for_validation, attributes=attributes, receiving_service=self, - parent_sdk_version=attributes.get("version"), + parent_sdk_version=attributes["sender_sdk_version"], child_sdk_version=importlib.metadata.version("octue"), ) @@ -633,7 +633,7 @@ def _parse_question(self, question): event, attributes["question_uuid"], attributes["forward_logs"], - attributes["version"], + attributes["sender_sdk_version"], attributes["save_diagnostics"], originator, ) diff --git a/tests/cloud/pub_sub/test_events.py b/tests/cloud/pub_sub/test_events.py index 2d2ed3359..9442b4c38 100644 --- a/tests/cloud/pub_sub/test_events.py +++ b/tests/cloud/pub_sub/test_events.py @@ -649,7 +649,7 @@ def test_pull_and_enqueue_available_events(self): "sender_type": "CHILD", "order": 0, "question_uuid": question_uuid, - "version": "0.50.0", + "sender_sdk_version": "0.50.0", "originator_namespace": originator_namespace, "originator_name": originator_name, "originator_revision_tag": originator_revision_tag, From f61f38a5f1ac10d7478770a137c9d66e36291511 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 28 Mar 2024 16:06:15 +0000 Subject: [PATCH 080/169] FIX: Avoid getting earliest waiting message if there isn't one --- octue/cloud/events/replayer.py | 6 ++++-- octue/cloud/pub_sub/events.py | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/octue/cloud/events/replayer.py b/octue/cloud/events/replayer.py index 13e3d2a5c..8f83c5da6 100644 --- a/octue/cloud/events/replayer.py +++ b/octue/cloud/events/replayer.py @@ -51,8 +51,10 @@ def handle_events(self, events): for event in events: self._extract_and_enqueue_event(event) - self._earliest_waiting_event_number = min(self.waiting_events.keys()) - return self._attempt_to_handle_waiting_events() + # Handle the case where no events (or no valid events) have been received. + if self.waiting_events: + self._earliest_waiting_event_number = min(self.waiting_events.keys()) + return self._attempt_to_handle_waiting_events() def _extract_event_and_attributes(self, container): """Extract an event and its attributes from the event container. diff --git a/octue/cloud/pub_sub/events.py b/octue/cloud/pub_sub/events.py index 8423009b1..e7cdca682 100644 --- a/octue/cloud/pub_sub/events.py +++ b/octue/cloud/pub_sub/events.py @@ -252,6 +252,7 @@ def _pull_and_enqueue_available_events(self, timeout): for event in pull_response.received_messages: self._extract_and_enqueue_event(event) + # Handle the case where no events (or no valid events) have been received. if self.waiting_events: self._earliest_waiting_event_number = min(self.waiting_events.keys()) From 562d2b41003fc8f791b6d40a6f33824c855162aa Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 28 Mar 2024 16:08:48 +0000 Subject: [PATCH 081/169] FIX: Avoid silently failing if `question_uuid` attribute missing --- octue/cloud/events/handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octue/cloud/events/handler.py b/octue/cloud/events/handler.py index 292433cae..95d75f364 100644 --- a/octue/cloud/events/handler.py +++ b/octue/cloud/events/handler.py @@ -138,7 +138,7 @@ def _extract_and_enqueue_event(self, container): revision_tag=attributes["sender_revision_tag"], ) - self.question_uuid = attributes.get("question_uuid") + self.question_uuid = attributes["question_uuid"] self._child_sdk_version = attributes["sender_sdk_version"] logger.debug("%r received an event related to question %r.", self.receiving_service, self.question_uuid) From 95da7cd251c4e71aadad236e3979af61712844fd Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 28 Mar 2024 16:28:17 +0000 Subject: [PATCH 082/169] REF: Move `OCTUE_SERVICES_NAMESPACE` to package root --- octue/__init__.py | 2 ++ octue/cloud/pub_sub/subscription.py | 2 +- octue/cloud/pub_sub/topic.py | 2 +- octue/cloud/service_id.py | 3 --- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/octue/__init__.py b/octue/__init__.py index 138e91f2c..2082daaa9 100644 --- a/octue/__init__.py +++ b/octue/__init__.py @@ -6,6 +6,8 @@ __all__ = ("Runner",) + +OCTUE_SERVICES_NAMESPACE = "octue.services" REPOSITORY_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) diff --git a/octue/cloud/pub_sub/subscription.py b/octue/cloud/pub_sub/subscription.py index 021843d3f..8ed602a55 100644 --- a/octue/cloud/pub_sub/subscription.py +++ b/octue/cloud/pub_sub/subscription.py @@ -13,7 +13,7 @@ UpdateSubscriptionRequest, ) -from octue.cloud.service_id import OCTUE_SERVICES_NAMESPACE +from octue import OCTUE_SERVICES_NAMESPACE from octue.exceptions import ConflictingSubscriptionType diff --git a/octue/cloud/pub_sub/topic.py b/octue/cloud/pub_sub/topic.py index cb0ebda83..0ada554f0 100644 --- a/octue/cloud/pub_sub/topic.py +++ b/octue/cloud/pub_sub/topic.py @@ -6,7 +6,7 @@ from google.cloud.pubsub_v1 import PublisherClient from google.pubsub_v1.types.pubsub import Topic as Topic_ -from octue.cloud.service_id import OCTUE_SERVICES_NAMESPACE +from octue import OCTUE_SERVICES_NAMESPACE logger = logging.getLogger(__name__) diff --git a/octue/cloud/service_id.py b/octue/cloud/service_id.py index fa2e021a8..60852e82b 100644 --- a/octue/cloud/service_id.py +++ b/octue/cloud/service_id.py @@ -10,9 +10,6 @@ logger = logging.getLogger(__name__) - -OCTUE_SERVICES_NAMESPACE = "octue.services" - SERVICE_NAMESPACE_AND_NAME_PATTERN = r"([a-z0-9])+(-([a-z0-9])+)*" COMPILED_SERVICE_NAMESPACE_AND_NAME_PATTERN = re.compile(SERVICE_NAMESPACE_AND_NAME_PATTERN) From 76127e9c48f4a1af9e061fc49cc8fc32102f539a Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 28 Mar 2024 16:29:57 +0000 Subject: [PATCH 083/169] FEA: Allow specification of services namespace via `octue.yaml` --- octue/configuration.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/octue/configuration.py b/octue/configuration.py index 1a1f2f7d1..21937ef67 100644 --- a/octue/configuration.py +++ b/octue/configuration.py @@ -4,6 +4,8 @@ import yaml +from octue import OCTUE_SERVICES_NAMESPACE + logger = logging.getLogger(__name__) @@ -18,6 +20,7 @@ class ServiceConfiguration: :param str|None app_configuration_path: the path to the app configuration file containing configuration data for the service; if this is `None`, the default application configuration is used :param str|None diagnostics_cloud_path: the path to a cloud directory to store diagnostics (this includes the configuration, input values and manifest, and logs) :param iter(dict)|None service_registries: the names and endpoints of the registries used to resolve service revisions when asking questions; these should be in priority order (highest priority first) + :param str services_namespace: the services namespace to emit and consume events from :param str|None directory: if provided, find the app source, twine, and app configuration relative to this directory :return None: """ @@ -31,11 +34,15 @@ def __init__( app_configuration_path=None, diagnostics_cloud_path=None, service_registries=None, + services_namespace=OCTUE_SERVICES_NAMESPACE, directory=None, **kwargs, ): self.name = name self.namespace = namespace + self.diagnostics_cloud_path = diagnostics_cloud_path + self.service_registries = service_registries + self.services_namespace = services_namespace if directory: directory = os.path.abspath(directory) @@ -61,9 +68,6 @@ def __init__( else: self.app_configuration_path = None - self.diagnostics_cloud_path = diagnostics_cloud_path - self.service_registries = service_registries - if kwargs: logger.warning(f"The following keyword arguments were not used by {type(self).__name__}: {kwargs!r}.") From 9e7a772ba8565bf5126b919d5bea8d40ba0b7f6c Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 28 Mar 2024 16:32:34 +0000 Subject: [PATCH 084/169] REF: Remove adding of services namespace to topics and subscriptions --- octue/cloud/pub_sub/subscription.py | 7 +------ octue/cloud/pub_sub/topic.py | 8 +------- tests/cloud/pub_sub/test_subscription.py | 8 +------- tests/cloud/pub_sub/test_topic.py | 10 +--------- 4 files changed, 4 insertions(+), 29 deletions(-) diff --git a/octue/cloud/pub_sub/subscription.py b/octue/cloud/pub_sub/subscription.py index 8ed602a55..c9920be03 100644 --- a/octue/cloud/pub_sub/subscription.py +++ b/octue/cloud/pub_sub/subscription.py @@ -13,7 +13,6 @@ UpdateSubscriptionRequest, ) -from octue import OCTUE_SERVICES_NAMESPACE from octue.exceptions import ConflictingSubscriptionType @@ -55,11 +54,7 @@ def __init__( push_endpoint=None, bigquery_table_id=None, ): - if not name.startswith(OCTUE_SERVICES_NAMESPACE): - self.name = f"{OCTUE_SERVICES_NAMESPACE}.{name}" - else: - self.name = name - + self.name = name self.topic = topic self.filter = filter self.path = self.generate_subscription_path(project_name or self.topic.project_name, self.name) diff --git a/octue/cloud/pub_sub/topic.py b/octue/cloud/pub_sub/topic.py index 0ada554f0..0af113c99 100644 --- a/octue/cloud/pub_sub/topic.py +++ b/octue/cloud/pub_sub/topic.py @@ -6,8 +6,6 @@ from google.cloud.pubsub_v1 import PublisherClient from google.pubsub_v1.types.pubsub import Topic as Topic_ -from octue import OCTUE_SERVICES_NAMESPACE - logger = logging.getLogger(__name__) @@ -23,11 +21,7 @@ class Topic: """ def __init__(self, name, project_name): - if not name.startswith(OCTUE_SERVICES_NAMESPACE): - self.name = f"{OCTUE_SERVICES_NAMESPACE}.{name}" - else: - self.name = name - + self.name = name self.project_name = project_name self.path = self.generate_topic_path(self.project_name, self.name) self.messages_published = 0 diff --git a/tests/cloud/pub_sub/test_subscription.py b/tests/cloud/pub_sub/test_subscription.py index 903247a15..bbf4961c8 100644 --- a/tests/cloud/pub_sub/test_subscription.py +++ b/tests/cloud/pub_sub/test_subscription.py @@ -17,13 +17,7 @@ class TestSubscription(BaseTestCase): def test_repr(self): """Test that subscriptions are represented correctly.""" - self.assertEqual(repr(self.subscription), "") - - def test_namespace_only_in_name_once(self): - """Test that the subscription's namespace only appears in its name once, even if it is repeated.""" - self.assertEqual(self.subscription.name, "octue.services.world") - subscription_with_repeated_namespace = Subscription(name="octue.services.world", topic=self.topic) - self.assertEqual(subscription_with_repeated_namespace.name, "octue.services.world") + self.assertEqual(repr(self.subscription), "") def test_create_without_allow_existing_when_subscription_already_exists(self): """Test that an error is raised when trying to create a subscription that already exists and `allow_existing` is diff --git a/tests/cloud/pub_sub/test_topic.py b/tests/cloud/pub_sub/test_topic.py index 03fb05df6..92802e51e 100644 --- a/tests/cloud/pub_sub/test_topic.py +++ b/tests/cloud/pub_sub/test_topic.py @@ -10,15 +10,7 @@ class TestTopic(BaseTestCase): def test_repr(self): """Test that Topics are represented correctly.""" topic = Topic(name="world", project_name="my-project") - self.assertEqual(repr(topic), "") - - def test_namespace_only_in_name_once(self): - """Test that the topic's namespace only appears in its name once, even if it is repeated.""" - topic = Topic(name="world", project_name="my-project") - self.assertEqual(topic.name, "octue.services.world") - - topic_with_repeated_namespace = Topic(name="octue.services.world", project_name="my-project") - self.assertEqual(topic_with_repeated_namespace.name, "octue.services.world") + self.assertEqual(repr(topic), "") def test_create(self): """Test that a topic can be created and that it's marked as having its creation triggered locally.""" From da93219a8552aedfdce50e246bb1de32231dfcbf Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 28 Mar 2024 16:34:35 +0000 Subject: [PATCH 085/169] REF: Rename `OCTUE_SERVICES_NAMESPACE` --- octue/__init__.py | 2 +- octue/configuration.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/octue/__init__.py b/octue/__init__.py index 2082daaa9..a2acacf99 100644 --- a/octue/__init__.py +++ b/octue/__init__.py @@ -7,7 +7,7 @@ __all__ = ("Runner",) -OCTUE_SERVICES_NAMESPACE = "octue.services" +DEFAULT_OCTUE_SERVICES_NAMESPACE = "octue.services" REPOSITORY_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) diff --git a/octue/configuration.py b/octue/configuration.py index 21937ef67..540815aa7 100644 --- a/octue/configuration.py +++ b/octue/configuration.py @@ -4,7 +4,7 @@ import yaml -from octue import OCTUE_SERVICES_NAMESPACE +from octue import DEFAULT_OCTUE_SERVICES_NAMESPACE logger = logging.getLogger(__name__) @@ -34,7 +34,7 @@ def __init__( app_configuration_path=None, diagnostics_cloud_path=None, service_registries=None, - services_namespace=OCTUE_SERVICES_NAMESPACE, + services_namespace=DEFAULT_OCTUE_SERVICES_NAMESPACE, directory=None, **kwargs, ): From 059cab3be9b466aff03ee589202cf91f07b6a591 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 28 Mar 2024 17:00:21 +0000 Subject: [PATCH 086/169] ENH: Add services namespace to `GCPPubSubBackend` --- octue/resources/service_backends.py | 22 +++++++++++++--------- tests/resources/test_service_backends.py | 17 ++++++++++++----- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/octue/resources/service_backends.py b/octue/resources/service_backends.py index 9270b4a52..48916b9ce 100644 --- a/octue/resources/service_backends.py +++ b/octue/resources/service_backends.py @@ -36,23 +36,27 @@ class ServiceBackend(ABC): class GCPPubSubBackend(ServiceBackend): - """A dataclass containing the details needed to use Google Cloud Platform Pub/Sub as a Service backend. + """A dataclass containing the details needed to use Google Cloud Pub/Sub as a Service backend. - :param str project_name: + :param str project_name: the name of the project to use for Pub/Sub + :param str services_namespace: the name of the topic to publish/subscribe events to/from :return None: """ - def __init__(self, project_name): - if project_name is None: - raise exceptions.CloudLocationNotSpecified( - "The project name must be specified for a service to connect to the correct Google Cloud Pub/Sub " - f"instance - it's currently {project_name!r}.", - ) + ERROR_MESSAGE = ( + "`project_name` and `services_namespace` must be specified for a service to connect to the correct service - " + "one of these is currently None." + ) + + def __init__(self, project_name, services_namespace): + if project_name is None or services_namespace is None: + raise exceptions.CloudLocationNotSpecified(self.ERROR_MESSAGE) self.project_name = project_name + self.services_namespace = services_namespace def __repr__(self): - return f"<{type(self).__name__}(project_name={self.project_name!r})>" + return f"<{type(self).__name__}(project_name={self.project_name!r}, services_namespace={self.services_namespace!r})>" AVAILABLE_BACKENDS = { diff --git a/tests/resources/test_service_backends.py b/tests/resources/test_service_backends.py index e0cabb357..d61ff997e 100644 --- a/tests/resources/test_service_backends.py +++ b/tests/resources/test_service_backends.py @@ -16,9 +16,16 @@ def test_existing_backend_can_be_retrieved(self): def test_repr(self): """Test the representation displays as expected.""" - self.assertEqual(repr(GCPPubSubBackend(project_name="hello")), "") + self.assertEqual( + repr(GCPPubSubBackend(project_name="hello", services_namespace="world")), + "", + ) - def test_error_raised_if_project_name_is_none(self): - """Test that an error is raised if the project name is not given during `GCPPubSubBackend` instantiation.""" - with self.assertRaises(CloudLocationNotSpecified): - GCPPubSubBackend(project_name=None) + def test_error_raised_if_project_name_or_services_namespace_is_none(self): + """Test that an error is raised if the project name or services namespace aren't given during `GCPPubSubBackend` + instantiation. + """ + for project_name, services_namespace in (("hello", None), (None, "world")): + with self.subTest(project_name=project_name, services_namespace=services_namespace): + with self.assertRaises(CloudLocationNotSpecified): + GCPPubSubBackend(project_name=project_name, services_namespace=services_namespace) From 263377ea57df294bd4f32e3635097e283fac07e7 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 28 Mar 2024 17:20:27 +0000 Subject: [PATCH 087/169] FEA: Publish/subscribe to single topic in `Service` BREAKING CHANGE: Upgrade all services in your network to this version or newer. skipci --- octue/cloud/pub_sub/service.py | 54 ++++++++++++++++------------------ 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index 54dc599ff..bc73271a3 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -94,6 +94,11 @@ def __init__(self, backend, service_id=None, run_function=None, name=None, servi self._publisher = None self._event_handler = None + self.topic = Topic(name=self.backend.services_namespace, project_name=self.backend.project_name) + + if not self.topic.exists(): + raise octue.exceptions.ServiceNotFound(f"Topic {self.backend.services_namespace} cannot be found.") + def __repr__(self): """Represent the service as a string. @@ -137,17 +142,15 @@ def serve(self, timeout=None, delete_topic_and_subscription_on_exit=False, allow :return (google.cloud.pubsub_v1.subscriber.futures.StreamingPullFuture, google.cloud.pubsub_v1.SubscriberClient): """ logger.info("Starting %r.", self) - topic = Topic(name=self._pub_sub_id, project_name=self.backend.project_name) subscription = Subscription( - name=self._pub_sub_id, - topic=topic, - filter=f'attributes.sender_type = "{PARENT_SENDER_TYPE}"', + name=".".join((self.backend.services_namespace, self._pub_sub_id)), + topic=self.topic, + filter=f'attributes.recipient = "{self.id}" AND attributes.sender_type = "{PARENT_SENDER_TYPE}"', expiration_time=None, ) try: - topic.create(allow_existing=allow_existing) subscription.create(allow_existing=allow_existing) except google.api_core.exceptions.AlreadyExists: raise octue.exceptions.ServiceAlreadyExists(f"A service with the ID {self.id!r} already exists.") @@ -178,11 +181,8 @@ def serve(self, timeout=None, delete_topic_and_subscription_on_exit=False, allow if subscription.creation_triggered_locally: subscription.delete() - if topic.creation_triggered_locally: - topic.delete() - except Exception: - logger.error("Deletion of topic and/or subscription %r failed.", topic.name) + logger.error("Deletion of subscription %r failed.", subscription.name) subscriber.close() @@ -210,16 +210,15 @@ def answer(self, question, heartbeat_interval=120, timeout=30): except jsonschema.ValidationError: return - topic = Topic(name=self._pub_sub_id, project_name=self.backend.project_name) heartbeater = None try: - self._send_delivery_acknowledgment(topic, question_uuid, originator) + self._send_delivery_acknowledgment(self.topic, question_uuid, originator) heartbeater = RepeatingTimer( interval=heartbeat_interval, function=self._send_heartbeat, - kwargs={"topic": topic, "question_uuid": question_uuid, "originator": originator}, + kwargs={"topic": self.topic, "question_uuid": question_uuid, "originator": originator}, ) heartbeater.daemon = True @@ -228,7 +227,7 @@ def answer(self, question, heartbeat_interval=120, timeout=30): if forward_logs: analysis_log_handler = GoogleCloudPubSubHandler( message_sender=self._send_message, - topic=topic, + topic=self.topic, question_uuid=question_uuid, originator=originator, ) @@ -243,7 +242,7 @@ def answer(self, question, heartbeat_interval=120, timeout=30): analysis_log_handler=analysis_log_handler, handle_monitor_message=functools.partial( self._send_monitor_message, - topic=topic, + topic=self.topic, question_uuid=question_uuid, originator=originator, ), @@ -257,7 +256,7 @@ def answer(self, question, heartbeat_interval=120, timeout=30): self._send_message( message=result, - topic=topic, + topic=self.topic, originator=originator, attributes={"question_uuid": question_uuid, "sender_type": CHILD_SENDER_TYPE}, timeout=timeout, @@ -271,7 +270,7 @@ def answer(self, question, heartbeat_interval=120, timeout=30): heartbeater.cancel() warn_if_incompatible(child_sdk_version=self._local_sdk_version, parent_sdk_version=parent_sdk_version) - self.send_exception(topic, question_uuid, originator, timeout=timeout) + self.send_exception(self.topic, question_uuid, originator, timeout=timeout) raise error def ask( @@ -330,20 +329,22 @@ def ask( "the new cloud locations." ) - topic = Topic(name=convert_service_id_to_pub_sub_form(service_id), project_name=self.backend.project_name) - - if not topic.exists(timeout=timeout): - raise octue.exceptions.ServiceNotFound(f"Service with ID {service_id!r} cannot be found.") - question_uuid = question_uuid or str(uuid.uuid4()) if asynchronous and not push_endpoint: answer_subscription = None else: + pub_sub_id = convert_service_id_to_pub_sub_form(service_id) + answer_subscription = Subscription( - name=".".join((topic.name, ANSWERS_NAMESPACE, question_uuid)), - topic=topic, - filter=f'attributes.question_uuid = "{question_uuid}" AND attributes.sender_type = "{CHILD_SENDER_TYPE}"', + name=".".join((self.backend.services_namespace, pub_sub_id, ANSWERS_NAMESPACE, question_uuid)), + topic=self.topic, + filter=( + f'attributes.sender = "{service_id}" ' + f'attributes.recipient = "{self.id}" ' + f'AND attributes.question_uuid = "{question_uuid}" ' + f'AND attributes.sender_type = "{CHILD_SENDER_TYPE}"' + ), push_endpoint=push_endpoint, ) answer_subscription.create(allow_existing=False) @@ -355,7 +356,6 @@ def ask( service_id=service_id, forward_logs=subscribe_to_logs, save_diagnostics=save_diagnostics, - topic=topic, question_uuid=question_uuid, ) @@ -494,7 +494,6 @@ def _send_question( service_id, forward_logs, save_diagnostics, - topic, question_uuid, timeout=30, ): @@ -506,7 +505,6 @@ def _send_question( :param str service_id: the ID of the child to send the question to :param bool forward_logs: whether to request the child to forward its logs :param str save_diagnostics: must be one of {"SAVE_DIAGNOSTICS_OFF", "SAVE_DIAGNOSTICS_ON_CRASH", "SAVE_DIAGNOSTICS_ON"}; if turned on, allow the input values and manifest (and its datasets) to be saved by the child either all the time or just if it fails while processing them - :param octue.cloud.pub_sub.topic.Topic topic: topic to send the acknowledgement to :param str question_uuid: the UUID of the question being sent :param float timeout: time in seconds after which to give up sending :return None: @@ -519,7 +517,7 @@ def _send_question( future = self._send_message( message=question, - topic=topic, + topic=self.topic, timeout=timeout, originator=self.id, attributes={ From 6ec4f5e9515b8788c33b7f7220d0ea2fe8297151 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 28 Mar 2024 18:05:30 +0000 Subject: [PATCH 088/169] REF: Switch back to unsplit SRUIDs in event attributes --- octue/cloud/emulators/_pub_sub.py | 19 +++++------------- octue/cloud/events/handler.py | 8 +------- octue/cloud/pub_sub/events.py | 14 ++++--------- octue/cloud/pub_sub/service.py | 33 +++++-------------------------- 4 files changed, 15 insertions(+), 59 deletions(-) diff --git a/octue/cloud/emulators/_pub_sub.py b/octue/cloud/emulators/_pub_sub.py index dbd1b40bd..bf1a6de12 100644 --- a/octue/cloud/emulators/_pub_sub.py +++ b/octue/cloud/emulators/_pub_sub.py @@ -6,7 +6,7 @@ from octue.cloud.pub_sub import Subscription, Topic from octue.cloud.pub_sub.service import ANSWERS_NAMESPACE, PARENT_SENDER_TYPE, Service -from octue.cloud.service_id import convert_service_id_to_pub_sub_form, split_service_id +from octue.cloud.service_id import convert_service_id_to_pub_sub_form from octue.resources import Manifest from octue.utils.dictionaries import make_minimal_dictionary from octue.utils.encoders import OctueJSONEncoder @@ -366,9 +366,6 @@ def ask( if input_manifest is not None: question["input_manifest"] = input_manifest.to_primitive() - originator_namespace, originator_name, originator_revision_tag = split_service_id(self.id) - recipient_namespace, recipient_name, recipient_revision_tag = split_service_id(service_id) - try: self.children[service_id].answer( MockMessage.from_primitive( @@ -376,19 +373,13 @@ def ask( attributes={ "question_uuid": question_uuid, "forward_logs": subscribe_to_logs, - "sender_sdk_version": parent_sdk_version, "save_diagnostics": save_diagnostics, "order": 0, - "originator_namespace": originator_namespace, - "originator_name": originator_name, - "originator_revision_tag": originator_revision_tag, - "sender_namespace": originator_namespace, - "sender_name": originator_name, - "sender_revision_tag": originator_revision_tag, + "originator": self.id, + "sender": self.id, "sender_type": PARENT_SENDER_TYPE, - "recipient_namespace": recipient_namespace, - "recipient_name": recipient_name, - "recipient_revision_tag": recipient_revision_tag, + "sender_sdk_version": parent_sdk_version, + "recipient": service_id, }, ) ) diff --git a/octue/cloud/events/handler.py b/octue/cloud/events/handler.py index 95d75f364..34648ef1c 100644 --- a/octue/cloud/events/handler.py +++ b/octue/cloud/events/handler.py @@ -9,7 +9,6 @@ from octue.cloud import EXCEPTIONS_MAPPING from octue.cloud.events.validation import SERVICE_COMMUNICATION_SCHEMA, is_event_valid -from octue.cloud.service_id import create_sruid from octue.definitions import GOOGLE_COMPUTE_PROVIDERS from octue.log_handlers import COLOUR_PALETTE from octue.resources.manifest import Manifest @@ -132,12 +131,7 @@ def _extract_and_enqueue_event(self, container): # Get the child's SRUID and Octue SDK version from the first event. if not self._child_sdk_version: - self.child_sruid = create_sruid( - namespace=attributes["sender_namespace"], - name=attributes["sender_name"], - revision_tag=attributes["sender_revision_tag"], - ) - + self.child_sruid = attributes["sender"] self.question_uuid = attributes["question_uuid"] self._child_sdk_version = attributes["sender_sdk_version"] diff --git a/octue/cloud/pub_sub/events.py b/octue/cloud/pub_sub/events.py index e7cdca682..8aa8731ba 100644 --- a/octue/cloud/pub_sub/events.py +++ b/octue/cloud/pub_sub/events.py @@ -35,17 +35,11 @@ def extract_event_and_attributes_from_pub_sub_message(message): converted_attributes = { "question_uuid": attributes["question_uuid"], "order": int(attributes["order"]), - "sender_sdk_version": attributes["sender_sdk_version"], - "originator_namespace": attributes["originator_namespace"], - "originator_name": attributes["originator_name"], - "originator_revision_tag": attributes["originator_revision_tag"], - "sender_namespace": attributes["sender_namespace"], - "sender_name": attributes["sender_name"], - "sender_revision_tag": attributes["sender_revision_tag"], + "originator": attributes["originator"], + "sender": attributes["sender"], "sender_type": attributes["sender_type"], - "recipient_namespace": attributes["recipient_namespace"], - "recipient_name": attributes["recipient_name"], - "recipient_revision_tag": attributes["recipient_revision_tag"], + "sender_sdk_version": attributes["sender_sdk_version"], + "recipient": attributes["recipient"], } # Required for question events. diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index bc73271a3..99018276d 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -22,7 +22,6 @@ convert_service_id_to_pub_sub_form, create_sruid, get_default_sruid, - get_sruid_from_pub_sub_resource_name, raise_if_revision_not_registered, split_service_id, validate_sruid, @@ -97,7 +96,7 @@ def __init__(self, backend, service_id=None, run_function=None, name=None, servi self.topic = Topic(name=self.backend.services_namespace, project_name=self.backend.project_name) if not self.topic.exists(): - raise octue.exceptions.ServiceNotFound(f"Topic {self.backend.services_namespace} cannot be found.") + raise octue.exceptions.ServiceNotFound(f"Topic {self.backend.services_namespace!r} cannot be found.") def __repr__(self): """Represent the service as a string. @@ -440,25 +439,9 @@ def _send_message(self, message, topic, originator, attributes=None, timeout=30) :return google.cloud.pubsub_v1.publisher.futures.Future: """ attributes = attributes or {} - - originator_namespace, originator_name, originator_revision_tag = split_service_id(originator) - sender_namespace, sender_name, sender_revision_tag = split_service_id(self.id) - - recipient_namespace, recipient_name, recipient_revision_tag = split_service_id( - get_sruid_from_pub_sub_resource_name(topic.name) - ) - - attributes["originator_namespace"] = originator_namespace - attributes["originator_name"] = originator_name - attributes["originator_revision_tag"] = originator_revision_tag - - attributes["sender_namespace"] = sender_namespace - attributes["sender_name"] = sender_name - attributes["sender_revision_tag"] = sender_revision_tag - - attributes["recipient_namespace"] = recipient_namespace - attributes["recipient_name"] = recipient_name - attributes["recipient_revision_tag"] = recipient_revision_tag + attributes["originator"] = originator + attributes["sender"] = self.id + attributes["recipient"] = originator with send_message_lock: attributes["sender_sdk_version"] = self._local_sdk_version @@ -621,17 +604,11 @@ def _parse_question(self, question): logger.info("%r parsed the question successfully.", self) - originator = create_sruid( - namespace=attributes["originator_namespace"], - name=attributes["originator_name"], - revision_tag=attributes["originator_revision_tag"], - ) - return ( event, attributes["question_uuid"], attributes["forward_logs"], attributes["sender_sdk_version"], attributes["save_diagnostics"], - originator, + attributes["originator"], ) From 22befccf25b72de60a3aeae59941c922cab89937 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 28 Mar 2024 18:48:58 +0000 Subject: [PATCH 089/169] FIX: Add correct recipient to event attributes --- octue/cloud/pub_sub/logging.py | 5 ++++- octue/cloud/pub_sub/service.py | 12 ++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/octue/cloud/pub_sub/logging.py b/octue/cloud/pub_sub/logging.py index 902e8773c..4e091dc84 100644 --- a/octue/cloud/pub_sub/logging.py +++ b/octue/cloud/pub_sub/logging.py @@ -12,15 +12,17 @@ class GoogleCloudPubSubHandler(logging.Handler): :param octue.cloud.pub_sub.topic.Topic topic: topic to publish log records to :param str question_uuid: the UUID of the question to handle log records for :param str originator: the SRUID of the service that asked the question these log records are related to + :param str recipient: the SRUID of the service to send these log records to :param float timeout: timeout in seconds for attempting to publish each log record :return None: """ - def __init__(self, message_sender, topic, question_uuid, originator, timeout=60, *args, **kwargs): + def __init__(self, message_sender, topic, question_uuid, originator, recipient, timeout=60, *args, **kwargs): super().__init__(*args, **kwargs) self.topic = topic self.question_uuid = question_uuid self.originator = originator + self.recipient = recipient self.timeout = timeout self._send_message = message_sender @@ -38,6 +40,7 @@ def emit(self, record): }, topic=self.topic, originator=self.originator, + recipient=self.recipient, attributes={ "question_uuid": self.question_uuid, "sender_type": "CHILD", # The sender type is repeated here as a string to avoid a circular import. diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index 99018276d..f3b35ce6f 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -229,6 +229,7 @@ def answer(self, question, heartbeat_interval=120, timeout=30): topic=self.topic, question_uuid=question_uuid, originator=originator, + recipient=originator, ) else: analysis_log_handler = None @@ -257,6 +258,7 @@ def answer(self, question, heartbeat_interval=120, timeout=30): message=result, topic=self.topic, originator=originator, + recipient=originator, attributes={"question_uuid": question_uuid, "sender_type": CHILD_SENDER_TYPE}, timeout=timeout, ) @@ -423,17 +425,19 @@ def send_exception(self, topic, question_uuid, originator, timeout=30): }, topic=topic, originator=originator, + recipient=originator, attributes={"question_uuid": question_uuid, "sender_type": CHILD_SENDER_TYPE}, timeout=timeout, ) - def _send_message(self, message, topic, originator, attributes=None, timeout=30): + def _send_message(self, message, topic, originator, recipient, attributes=None, timeout=30): """Send a JSON-serialised message to the given topic with optional message attributes and increment the `messages_published` attribute of the topic by one. This method is thread-safe. :param dict message: JSON-serialisable data to send as a message :param octue.cloud.pub_sub.topic.Topic topic: the Pub/Sub topic to send the message to :param str originator: the SRUID of the service that asked the question this event is related to + :param str recipient: :param dict|None attributes: key-value pairs to attach to the message - the values must be strings or bytes :param int|float timeout: the timeout for sending the message in seconds :return google.cloud.pubsub_v1.publisher.futures.Future: @@ -441,7 +445,7 @@ def _send_message(self, message, topic, originator, attributes=None, timeout=30) attributes = attributes or {} attributes["originator"] = originator attributes["sender"] = self.id - attributes["recipient"] = originator + attributes["recipient"] = recipient with send_message_lock: attributes["sender_sdk_version"] = self._local_sdk_version @@ -503,6 +507,7 @@ def _send_question( topic=self.topic, timeout=timeout, originator=self.id, + recipient=service_id, attributes={ "question_uuid": question_uuid, "forward_logs": forward_logs, @@ -532,6 +537,7 @@ def _send_delivery_acknowledgment(self, topic, question_uuid, originator, timeou topic=topic, timeout=timeout, originator=originator, + recipient=originator, attributes={"question_uuid": question_uuid, "sender_type": CHILD_SENDER_TYPE}, ) @@ -553,6 +559,7 @@ def _send_heartbeat(self, topic, question_uuid, originator, timeout=30): }, topic=topic, originator=originator, + recipient=originator, timeout=timeout, attributes={"question_uuid": question_uuid, "sender_type": CHILD_SENDER_TYPE}, ) @@ -573,6 +580,7 @@ def _send_monitor_message(self, data, topic, question_uuid, originator, timeout= {"kind": "monitor_message", "data": data}, topic=topic, originator=originator, + recipient=originator, timeout=timeout, attributes={"question_uuid": question_uuid, "sender_type": CHILD_SENDER_TYPE}, ) From 974f52cc42cfacf55436e1444e106f8b96fde04f Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 28 Mar 2024 18:49:17 +0000 Subject: [PATCH 090/169] FIX: Avoid circular imports --- octue/resources/service_backends.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/octue/resources/service_backends.py b/octue/resources/service_backends.py index 48916b9ce..438b448ad 100644 --- a/octue/resources/service_backends.py +++ b/octue/resources/service_backends.py @@ -48,7 +48,8 @@ class GCPPubSubBackend(ServiceBackend): "one of these is currently None." ) - def __init__(self, project_name, services_namespace): + # "octue.services" is repeated here to avoid a circular import. + def __init__(self, project_name, services_namespace="octue.services"): if project_name is None or services_namespace is None: raise exceptions.CloudLocationNotSpecified(self.ERROR_MESSAGE) From 7c7bf51c4a65094a9c84d3636e46f1a13d67f7af Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 28 Mar 2024 18:49:38 +0000 Subject: [PATCH 091/169] ENH: Update Pub/Sub emulators to use a single topic skipci --- octue/cloud/emulators/_pub_sub.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/octue/cloud/emulators/_pub_sub.py b/octue/cloud/emulators/_pub_sub.py index bf1a6de12..88f9816b6 100644 --- a/octue/cloud/emulators/_pub_sub.py +++ b/octue/cloud/emulators/_pub_sub.py @@ -1,12 +1,12 @@ import importlib.metadata import json import logging +from collections import defaultdict import google.api_core from octue.cloud.pub_sub import Subscription, Topic -from octue.cloud.pub_sub.service import ANSWERS_NAMESPACE, PARENT_SENDER_TYPE, Service -from octue.cloud.service_id import convert_service_id_to_pub_sub_form +from octue.cloud.pub_sub.service import PARENT_SENDER_TYPE, Service from octue.resources import Manifest from octue.utils.dictionaries import make_minimal_dictionary from octue.utils.encoders import OctueJSONEncoder @@ -14,8 +14,9 @@ logger = logging.getLogger(__name__) -TOPICS = {} -SUBSCRIPTIONS = {} +TOPICS = set() +SUBSCRIPTIONS = set() +MESSAGES = defaultdict(list) class MockTopic(Topic): @@ -33,7 +34,7 @@ def create(self, allow_existing=False): raise google.api_core.exceptions.AlreadyExists(f"Topic {self.path!r} already exists.") if not self.exists(): - TOPICS[self.name] = [] + TOPICS.add(self.name) self._created = True def delete(self): @@ -42,7 +43,7 @@ def delete(self): :return None: """ try: - del TOPICS[self.name] + TOPICS.remove(self.name) except KeyError: pass @@ -73,7 +74,7 @@ def create(self, allow_existing=False): raise google.api_core.exceptions.AlreadyExists(f"Subscription {self.path!r} already exists.") if not self.exists(): - SUBSCRIPTIONS[self.name] = [] + SUBSCRIPTIONS.add(self.name) self._created = True def delete(self): @@ -146,8 +147,7 @@ def publish(self, topic, data, retry=None, **attributes): :param google.api_core.retry.Retry|None retry: :return MockFuture: """ - subscription_name = ".".join((get_pub_sub_resource_name(topic), ANSWERS_NAMESPACE, attributes["question_uuid"])) - SUBSCRIPTIONS[subscription_name].append(MockMessage(data=data, attributes=attributes)) + MESSAGES[attributes["question_uuid"]].append(MockMessage(data=data, attributes=attributes)) return MockFuture() @@ -191,12 +191,12 @@ def pull(self, request, timeout=None, retry=None): if self.closed: raise ValueError("ValueError: Cannot invoke RPC: Channel closed!") - subscription_name = get_pub_sub_resource_name(request["subscription"]) + question_uuid = request["subscription"].split(".")[-1] try: return MockPullResponse( received_messages=[ - MockMessageWrapper(message=SUBSCRIPTIONS[subscription_name].pop(0)), + MockMessageWrapper(message=MESSAGES[question_uuid].pop(0)), ] ) @@ -356,8 +356,7 @@ def ask( # Delete question from messages sent to subscription so the parent doesn't pick it up as a response message. We # do this as subscription filtering isn't implemented in this set of mocks. - subscription_name = ".".join((convert_service_id_to_pub_sub_form(service_id), ANSWERS_NAMESPACE, question_uuid)) - SUBSCRIPTIONS["octue.services." + subscription_name].pop(0) + MESSAGES[question_uuid].pop(0) question = make_minimal_dictionary(kind="question", input_values=input_values, children=children) From 807ae752fe4f008dd94470164aaf463ec3b09f93 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 2 Apr 2024 14:46:15 +0100 Subject: [PATCH 092/169] TST: Update pub/sub event tests --- tests/cloud/pub_sub/test_events.py | 159 +++++++++++++++-------------- 1 file changed, 81 insertions(+), 78 deletions(-) diff --git a/tests/cloud/pub_sub/test_events.py b/tests/cloud/pub_sub/test_events.py index 9442b4c38..386f6dd56 100644 --- a/tests/cloud/pub_sub/test_events.py +++ b/tests/cloud/pub_sub/test_events.py @@ -3,44 +3,41 @@ import uuid from unittest.mock import patch -from octue.cloud.emulators._pub_sub import ( - SUBSCRIPTIONS, - MockMessage, - MockService, - MockSubscriber, - MockSubscription, - MockTopic, -) +from octue.cloud.emulators._pub_sub import MESSAGES, MockMessage, MockService, MockSubscription, MockTopic from octue.cloud.emulators.child import ServicePatcher from octue.cloud.pub_sub.events import GoogleCloudPubSubEventHandler -from octue.cloud.service_id import split_service_id from octue.resources.service_backends import GCPPubSubBackend from tests import TEST_PROJECT_NAME from tests.base import BaseTestCase -parent = MockService(service_id="my-org/my-service:1.0.0", backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) - - def create_mock_topic_and_subscription(): """Create a question UUID, mock topic, and mock subscription. :return (str, octue.cloud.emulators._pub_sub.MockTopic, octue.cloud.emulators._pub_sub.MockSubscription): question UUID, topic, and subscription """ question_uuid = str(uuid.uuid4()) - topic = MockTopic(name="my-org.my-service.1-0-0", project_name=TEST_PROJECT_NAME) - subscription = MockSubscription(name=f"my-org.my-service.1-0-0.answers.{question_uuid}", topic=topic) + topic = MockTopic(name="octue.services", project_name=TEST_PROJECT_NAME) + topic.create(allow_existing=True) + + subscription = MockSubscription(name=f"octue.services.my-org.my-service.1-0-0.answers.{question_uuid}", topic=topic) subscription.create() - return question_uuid, topic, subscription + + with ServicePatcher(): + parent = MockService( + service_id="my-org/my-service:1.0.0", backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME) + ) + + return question_uuid, topic, subscription, parent class TestPubSubEventHandler(BaseTestCase): def test_timeout(self): """Test that a TimeoutError is raised if message handling takes longer than the given timeout.""" - question_uuid, _, mock_subscription = create_mock_topic_and_subscription() + question_uuid, _, mock_subscription, parent = create_mock_topic_and_subscription() - with patch("octue.cloud.pub_sub.events.SubscriberClient", MockSubscriber): + with ServicePatcher(): event_handler = GoogleCloudPubSubEventHandler( subscription=mock_subscription, receiving_service=parent, @@ -53,9 +50,9 @@ def test_timeout(self): def test_in_order_messages_are_handled_in_order(self): """Test that messages received in order are handled in order.""" - question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() + question_uuid, mock_topic, mock_subscription, parent = create_mock_topic_and_subscription() - with patch("octue.cloud.pub_sub.events.SubscriberClient", MockSubscriber): + with ServicePatcher(): event_handler = GoogleCloudPubSubEventHandler( subscription=mock_subscription, receiving_service=parent, @@ -63,7 +60,7 @@ def test_in_order_messages_are_handled_in_order(self): schema={}, ) - child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) + child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) messages = [ {"event": {"kind": "test"}, "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}}, @@ -78,6 +75,7 @@ def test_in_order_messages_are_handled_in_order(self): attributes=message["attributes"], topic=mock_topic, originator=parent.id, + recipient=parent.id, ) result = event_handler.handle_events() @@ -90,9 +88,9 @@ def test_in_order_messages_are_handled_in_order(self): def test_out_of_order_messages_are_handled_in_order(self): """Test that messages received out of order are handled in order.""" - question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() + question_uuid, mock_topic, mock_subscription, parent = create_mock_topic_and_subscription() - with patch("octue.cloud.pub_sub.events.SubscriberClient", MockSubscriber): + with ServicePatcher(): event_handler = GoogleCloudPubSubEventHandler( subscription=mock_subscription, receiving_service=parent, @@ -100,7 +98,7 @@ def test_out_of_order_messages_are_handled_in_order(self): schema={}, ) - child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) + child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) messages = [ { @@ -128,6 +126,7 @@ def test_out_of_order_messages_are_handled_in_order(self): attributes=message["attributes"], topic=mock_topic, originator=parent.id, + recipient=parent.id, ) result = event_handler.handle_events() @@ -148,9 +147,9 @@ def test_out_of_order_messages_with_end_message_first_are_handled_in_order(self) """Test that messages received out of order and with the final message (the message that triggers a value to be returned) are handled in order. """ - question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() + question_uuid, mock_topic, mock_subscription, parent = create_mock_topic_and_subscription() - with patch("octue.cloud.pub_sub.events.SubscriberClient", MockSubscriber): + with ServicePatcher(): event_handler = GoogleCloudPubSubEventHandler( subscription=mock_subscription, receiving_service=parent, @@ -158,7 +157,7 @@ def test_out_of_order_messages_with_end_message_first_are_handled_in_order(self) schema={}, ) - child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) + child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) messages = [ { @@ -186,6 +185,7 @@ def test_out_of_order_messages_with_end_message_first_are_handled_in_order(self) attributes=message["attributes"], topic=mock_topic, originator=parent.id, + recipient=parent.id, ) result = event_handler.handle_events() @@ -204,9 +204,9 @@ def test_out_of_order_messages_with_end_message_first_are_handled_in_order(self) def test_no_timeout(self): """Test that message handling works with no timeout.""" - question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() + question_uuid, mock_topic, mock_subscription, parent = create_mock_topic_and_subscription() - with patch("octue.cloud.pub_sub.events.SubscriberClient", MockSubscriber): + with ServicePatcher(): event_handler = GoogleCloudPubSubEventHandler( subscription=mock_subscription, receiving_service=parent, @@ -214,7 +214,7 @@ def test_no_timeout(self): schema={}, ) - child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) + child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) messages = [ { @@ -237,6 +237,7 @@ def test_no_timeout(self): attributes=message["attributes"], topic=mock_topic, originator=parent.id, + recipient=parent.id, ) result = event_handler.handle_events(timeout=None) @@ -249,12 +250,11 @@ def test_no_timeout(self): def test_delivery_acknowledgement(self): """Test that a delivery acknowledgement message is handled correctly.""" - question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() + question_uuid, mock_topic, mock_subscription, parent = create_mock_topic_and_subscription() - with patch("octue.cloud.pub_sub.events.SubscriberClient", MockSubscriber): + with ServicePatcher(): event_handler = GoogleCloudPubSubEventHandler(subscription=mock_subscription, receiving_service=parent) - - child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) + child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) messages = [ { @@ -273,6 +273,7 @@ def test_delivery_acknowledgement(self): attributes=message["attributes"], topic=mock_topic, originator=parent.id, + recipient=parent.id, ) result = event_handler.handle_events() @@ -280,9 +281,9 @@ def test_delivery_acknowledgement(self): def test_error_raised_if_heartbeat_not_received_before_checked(self): """Test that an error is raised if a heartbeat isn't received before a heartbeat is first checked for.""" - question_uuid, _, mock_subscription = create_mock_topic_and_subscription() + question_uuid, _, mock_subscription, parent = create_mock_topic_and_subscription() - with patch("octue.cloud.pub_sub.events.SubscriberClient", MockSubscriber): + with ServicePatcher(): event_handler = GoogleCloudPubSubEventHandler(subscription=mock_subscription, receiving_service=parent) with self.assertRaises(TimeoutError) as error: @@ -293,9 +294,9 @@ def test_error_raised_if_heartbeat_not_received_before_checked(self): def test_error_raised_if_heartbeats_stop_being_received(self): """Test that an error is raised if heartbeats stop being received within the maximum interval.""" - question_uuid, _, mock_subscription = create_mock_topic_and_subscription() + question_uuid, _, mock_subscription, parent = create_mock_topic_and_subscription() - with patch("octue.cloud.pub_sub.events.SubscriberClient", MockSubscriber): + with ServicePatcher(): event_handler = GoogleCloudPubSubEventHandler(subscription=mock_subscription, receiving_service=parent) event_handler._last_heartbeat = datetime.datetime.now() - datetime.timedelta(seconds=30) @@ -307,15 +308,14 @@ def test_error_raised_if_heartbeats_stop_being_received(self): def test_error_not_raised_if_heartbeat_has_been_received_in_maximum_allowed_interval(self): """Test that an error is not raised if a heartbeat has been received in the maximum allowed interval.""" - question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() + question_uuid, mock_topic, mock_subscription, parent = create_mock_topic_and_subscription() - with patch("octue.cloud.pub_sub.events.SubscriberClient", MockSubscriber): + with ServicePatcher(): event_handler = GoogleCloudPubSubEventHandler(subscription=mock_subscription, receiving_service=parent) + child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) event_handler._last_heartbeat = datetime.datetime.now() - child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) - messages = [ { "event": { @@ -336,6 +336,7 @@ def test_error_not_raised_if_heartbeat_has_been_received_in_maximum_allowed_inte attributes=message["attributes"], topic=mock_topic, originator=parent.id, + recipient=parent.id, ) with patch( @@ -346,9 +347,9 @@ def test_error_not_raised_if_heartbeat_has_been_received_in_maximum_allowed_inte def test_time_since_last_heartbeat_is_none_if_no_heartbeat_received_yet(self): """Test that the time since the last heartbeat is `None` if no heartbeat has been received yet.""" - question_uuid, _, mock_subscription = create_mock_topic_and_subscription() + question_uuid, _, mock_subscription, parent = create_mock_topic_and_subscription() - with patch("octue.cloud.pub_sub.events.SubscriberClient", MockSubscriber): + with ServicePatcher(): event_handler = GoogleCloudPubSubEventHandler(subscription=mock_subscription, receiving_service=parent) self.assertIsNone(event_handler._time_since_last_heartbeat) @@ -357,13 +358,13 @@ def test_total_run_time_is_none_if_handle_events_has_not_been_called(self): """Test that the total run time for the message handler is `None` if the `handle_events` method has not been called. """ - question_uuid, _, mock_subscription = create_mock_topic_and_subscription() + question_uuid, _, mock_subscription, parent = create_mock_topic_and_subscription() event_handler = GoogleCloudPubSubEventHandler(subscription=mock_subscription, receiving_service=parent) self.assertIsNone(event_handler.total_run_time) def test_time_since_missing_message_is_none_if_no_unhandled_missing_messages(self): """Test that the `time_since_missing_message` property is `None` if there are no unhandled missing messages.""" - question_uuid, _, mock_subscription = create_mock_topic_and_subscription() + question_uuid, _, mock_subscription, parent = create_mock_topic_and_subscription() event_handler = GoogleCloudPubSubEventHandler(subscription=mock_subscription, receiving_service=parent) self.assertIsNone(event_handler.time_since_missing_event) @@ -371,9 +372,9 @@ def test_missing_messages_at_start_can_be_skipped(self): """Test that missing messages at the start of the event stream can be skipped if they aren't received after a given time period if subsequent messages have been received. """ - question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() + question_uuid, mock_topic, mock_subscription, parent = create_mock_topic_and_subscription() - with patch("octue.cloud.pub_sub.events.SubscriberClient", MockSubscriber): + with ServicePatcher(): event_handler = GoogleCloudPubSubEventHandler( subscription=mock_subscription, receiving_service=parent, @@ -382,11 +383,11 @@ def test_missing_messages_at_start_can_be_skipped(self): skip_missing_events_after=0, ) + child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) + # Simulate the first two messages not being received. mock_topic.messages_published = 2 - child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) - messages = [ { "event": {"kind": "test", "order": 2}, @@ -412,6 +413,7 @@ def test_missing_messages_at_start_can_be_skipped(self): attributes=message["attributes"], topic=mock_topic, originator=parent.id, + recipient=parent.id, ) result = event_handler.handle_events() @@ -429,9 +431,9 @@ def test_missing_messages_at_start_can_be_skipped(self): def test_missing_messages_in_middle_can_skipped(self): """Test that missing messages in the middle of the event stream can be skipped.""" - question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() + question_uuid, mock_topic, mock_subscription, parent = create_mock_topic_and_subscription() - with patch("octue.cloud.pub_sub.events.SubscriberClient", MockSubscriber): + with ServicePatcher(): event_handler = GoogleCloudPubSubEventHandler( subscription=mock_subscription, receiving_service=parent, @@ -440,7 +442,7 @@ def test_missing_messages_in_middle_can_skipped(self): skip_missing_events_after=0, ) - child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) + child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) # Send three consecutive messages. messages = [ @@ -464,6 +466,7 @@ def test_missing_messages_in_middle_can_skipped(self): attributes=message["attributes"], topic=mock_topic, originator=parent.id, + recipient=parent.id, ) # Simulate missing messages. @@ -475,6 +478,7 @@ def test_missing_messages_in_middle_can_skipped(self): attributes={"question_uuid": question_uuid, "sender_type": "CHILD"}, topic=mock_topic, originator=parent.id, + recipient=parent.id, ) event_handler.handle_events() @@ -492,9 +496,9 @@ def test_missing_messages_in_middle_can_skipped(self): def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): """Test that multiple blocks of missing messages in the middle of the event stream can be skipped.""" - question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() + question_uuid, mock_topic, mock_subscription, parent = create_mock_topic_and_subscription() - with patch("octue.cloud.pub_sub.events.SubscriberClient", MockSubscriber): + with ServicePatcher(): event_handler = GoogleCloudPubSubEventHandler( subscription=mock_subscription, receiving_service=parent, @@ -503,7 +507,7 @@ def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): skip_missing_events_after=0, ) - child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) + child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) # Send three consecutive messages. messages = [ @@ -523,7 +527,11 @@ def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): for message in messages: child._send_message( - message=message["event"], attributes=message["attributes"], topic=mock_topic, originator=parent.id + message=message["event"], + attributes=message["attributes"], + topic=mock_topic, + originator=parent.id, + recipient=parent.id, ) # Simulate missing messages. @@ -535,6 +543,7 @@ def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): attributes={"order": 5, "question_uuid": question_uuid, "sender_type": "CHILD"}, topic=mock_topic, originator=parent.id, + recipient=parent.id, ) # Simulate more missing messages. @@ -562,7 +571,11 @@ def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): for message in messages: child._send_message( - message=message["event"], attributes=message["attributes"], topic=mock_topic, originator=parent.id + message=message["event"], + attributes=message["attributes"], + topic=mock_topic, + originator=parent.id, + recipient=parent.id, ) event_handler.handle_events() @@ -584,9 +597,9 @@ def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): def test_all_messages_missing_apart_from_result(self): """Test that the result message is still handled if all other messages are missing.""" - question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() + question_uuid, mock_topic, mock_subscription, parent = create_mock_topic_and_subscription() - with patch("octue.cloud.pub_sub.events.SubscriberClient", MockSubscriber): + with ServicePatcher(): event_handler = GoogleCloudPubSubEventHandler( subscription=mock_subscription, receiving_service=parent, @@ -595,7 +608,7 @@ def test_all_messages_missing_apart_from_result(self): skip_missing_events_after=0, ) - child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) + child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) # Simulate missing messages. mock_topic.messages_published = 1000 @@ -606,6 +619,7 @@ def test_all_messages_missing_apart_from_result(self): attributes={"question_uuid": question_uuid, "sender_type": "CHILD"}, topic=mock_topic, originator=parent.id, + recipient=parent.id, ) event_handler.handle_events() @@ -617,7 +631,7 @@ def test_all_messages_missing_apart_from_result(self): class TestPullAndEnqueueAvailableMessages(BaseTestCase): def test_pull_and_enqueue_available_events(self): """Test that pulling and enqueuing a message works.""" - question_uuid, mock_topic, _ = create_mock_topic_and_subscription() + question_uuid, mock_topic, _, parent = create_mock_topic_and_subscription() with ServicePatcher(): mock_subscription = MockSubscription( @@ -640,25 +654,17 @@ def test_pull_and_enqueue_available_events(self): # Enqueue a mock message for a mock subscription to receive. mock_message = {"kind": "test"} - originator_namespace, originator_name, originator_revision_tag = split_service_id(parent.id) - - SUBSCRIPTIONS[mock_subscription.name] = [ + MESSAGES[question_uuid] = [ MockMessage.from_primitive( mock_message, attributes={ - "sender_type": "CHILD", "order": 0, "question_uuid": question_uuid, + "originator": parent.id, + "sender": parent.id, + "sender_type": "CHILD", "sender_sdk_version": "0.50.0", - "originator_namespace": originator_namespace, - "originator_name": originator_name, - "originator_revision_tag": originator_revision_tag, - "sender_namespace": originator_namespace, - "sender_name": originator_name, - "sender_revision_tag": originator_revision_tag, - "recipient_namespace": "my-org", - "recipient_name": "my-service", - "recipient_revision_tag": "1.0.0", + "recipient": "my-org/my-service:1.0.0", }, ) ] @@ -669,7 +675,7 @@ def test_pull_and_enqueue_available_events(self): def test_timeout_error_raised_if_result_message_not_received_in_time(self): """Test that a timeout error is raised if a result message is not received in time.""" - question_uuid, mock_topic, _ = create_mock_topic_and_subscription() + question_uuid, mock_topic, _, parent = create_mock_topic_and_subscription() with ServicePatcher(): mock_subscription = MockSubscription( @@ -687,9 +693,6 @@ def test_timeout_error_raised_if_result_message_not_received_in_time(self): event_handler.waiting_events = {} event_handler._start_time = 0 - # Create a mock subscription. - SUBSCRIPTIONS[mock_subscription.name] = [] - with self.assertRaises(TimeoutError): event_handler._pull_and_enqueue_available_events(timeout=1e-6) From f22b635143aa1495ae7a3fe45ecda021592981fa Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 2 Apr 2024 14:58:15 +0100 Subject: [PATCH 093/169] TST: Update pub/sub logging tests --- tests/cloud/pub_sub/test_logging.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/tests/cloud/pub_sub/test_logging.py b/tests/cloud/pub_sub/test_logging.py index fee4a5f01..a94c17a8c 100644 --- a/tests/cloud/pub_sub/test_logging.py +++ b/tests/cloud/pub_sub/test_logging.py @@ -3,9 +3,11 @@ from logging import makeLogRecord from unittest.mock import patch -from octue.cloud.emulators._pub_sub import SUBSCRIPTIONS, MockService, MockSubscription, MockTopic +from octue.cloud.emulators._pub_sub import MESSAGES, MockService, MockSubscription, MockTopic +from octue.cloud.emulators.child import ServicePatcher from octue.cloud.pub_sub.logging import GoogleCloudPubSubHandler from octue.resources.service_backends import GCPPubSubBackend +from tests import TEST_PROJECT_NAME from tests.base import BaseTestCase @@ -15,6 +17,13 @@ def __repr__(self): class TestGoogleCloudPubSubHandler(BaseTestCase): + @classmethod + def setUpClass(cls): + topic = MockTopic(name="octue.services", project_name=TEST_PROJECT_NAME) + + with ServicePatcher(): + topic.create(allow_existing=True) + def test_emit(self): """Test the log message is published when `GoogleCloudPubSubHandler.emit` is called.""" topic = MockTopic(name="octue.my-service.3-3-3", project_name="blah") @@ -27,17 +36,20 @@ def test_emit(self): log_record = makeLogRecord({"msg": "Starting analysis."}) backend = GCPPubSubBackend(project_name="blah") - service = MockService(backend=backend) + + with ServicePatcher(): + service = MockService(backend=backend) GoogleCloudPubSubHandler( message_sender=service._send_message, topic=topic, question_uuid=question_uuid, originator=service.id, + recipient="another/service:1.0.0", ).emit(log_record) self.assertEqual( - json.loads(SUBSCRIPTIONS[subscription.name][0].data.decode())["log_record"]["msg"], + json.loads(MESSAGES[question_uuid][0].data.decode())["log_record"]["msg"], "Starting analysis.", ) @@ -59,7 +71,9 @@ def test_emit_with_non_json_serialisable_args(self): ) backend = GCPPubSubBackend(project_name="blah") - service = MockService(backend=backend) + + with ServicePatcher(): + service = MockService(backend=backend) with patch("octue.cloud.emulators._pub_sub.MockPublisher.publish") as mock_publish: GoogleCloudPubSubHandler( @@ -67,6 +81,7 @@ def test_emit_with_non_json_serialisable_args(self): topic=topic, question_uuid="question-uuid", originator=service.id, + recipient="another/service:1.0.0", ).emit(record) self.assertEqual( From 166d1a2c2d7d770bcd96eeb15bd49c745d7757f3 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 2 Apr 2024 17:25:11 +0100 Subject: [PATCH 094/169] TST: Update service tests --- tests/cloud/pub_sub/test_service.py | 538 ++++++++++++++-------------- 1 file changed, 267 insertions(+), 271 deletions(-) diff --git a/tests/cloud/pub_sub/test_service.py b/tests/cloud/pub_sub/test_service.py index 1558e64af..0a1c403bc 100644 --- a/tests/cloud/pub_sub/test_service.py +++ b/tests/cloud/pub_sub/test_service.py @@ -15,7 +15,6 @@ DifferentMockAnalysis, MockAnalysis, MockAnalysisWithOutputManifest, - MockPullResponse, MockService, MockSubscription, MockTopic, @@ -43,14 +42,25 @@ class TestService(BaseTestCase): service_patcher = ServicePatcher() + @classmethod + def setUpClass(cls): + topic = MockTopic(name="octue.services", project_name=TEST_PROJECT_NAME) + + with cls.service_patcher: + topic.create(allow_existing=True) + def test_repr(self): """Test that services are represented as a string correctly.""" - service = Service(backend=BACKEND) + with self.service_patcher: + service = Service(backend=BACKEND) + self.assertEqual(repr(service), f"") def test_repr_with_name(self): """Test that services are represented using their name if they have one.""" - service = Service(backend=BACKEND, name=f"octue/blah-service:{MOCK_SERVICE_REVISION_TAG}") + with self.service_patcher: + service = Service(backend=BACKEND, name=f"octue/blah-service:{MOCK_SERVICE_REVISION_TAG}") + self.assertEqual(repr(service), f"") def test_service_id_cannot_be_non_none_empty_value(self): @@ -103,19 +113,6 @@ def test_serve_detached(self): self.assertFalse(future.returned) self.assertFalse(subscriber.closed) - def test_ask_on_non_existent_service_results_in_error(self): - """Test that trying to ask a question to a non-existent service (i.e. one without a topic in Google Pub/Sub) - results in an error. - """ - with patch("octue.cloud.pub_sub.service.Topic", new=MockTopic): - service = MockService(backend=BACKEND) - - with self.assertRaises(exceptions.ServiceNotFound): - service.ask( - service_id=f"my-org/existing-service:{MOCK_SERVICE_REVISION_TAG}", - input_values=[1, 2, 3, 4], - ) - def test_ask_unregistered_service_revision_when_service_registries_specified_results_in_error(self): """Test that an error is raised if attempting to ask an unregistered service a question when service registries are being used. @@ -171,15 +168,17 @@ def test_timeout_error_raised_if_no_messages_received_when_waiting(self): """Test that a TimeoutError is raised if no messages are received while waiting.""" mock_topic = MockTopic(name="amazing.service.9-9-9", project_name=TEST_PROJECT_NAME) mock_subscription = MockSubscription(name="amazing.service.9-9-9", topic=mock_topic) - service = Service(backend=BACKEND) - with patch("octue.cloud.pub_sub.service.pubsub_v1.SubscriberClient.pull", return_value=MockPullResponse()): + with self.service_patcher: + service = Service(backend=BACKEND) + with self.assertRaises(TimeoutError): service.wait_for_answer(subscription=mock_subscription, timeout=0.01) def test_error_raised_if_attempting_to_wait_for_answer_from_non_pull_subscription(self): """Test that an error is raised if attempting to wait for an answer from a push subscription.""" - service = Service(backend=BACKEND) + with self.service_patcher: + service = Service(backend=BACKEND) subscription = MockSubscription( name="world", @@ -192,13 +191,13 @@ def test_error_raised_if_attempting_to_wait_for_answer_from_non_pull_subscriptio def test_exceptions_in_responder_are_handled_and_sent_to_asker(self): """Test that exceptions raised in the child service are handled and sent back to the asker.""" - child = self.make_new_child_with_error( - twined.exceptions.InvalidManifestContents("'met_mast_id' is a required property") - ) + with self.service_patcher: + child = self.make_new_child_with_error( + twined.exceptions.InvalidManifestContents("'met_mast_id' is a required property") + ) - parent = MockService(backend=BACKEND, children={child.id: child}) + parent = MockService(backend=BACKEND, children={child.id: child}) - with self.service_patcher: child.serve() subscription, _ = parent.ask(service_id=child.id, input_values={}) @@ -211,10 +210,10 @@ def test_exceptions_with_multiple_arguments_in_responder_are_handled_and_sent_to """Test that exceptions with multiple arguments raised in the child service are handled and sent back to the asker. """ - child = self.make_new_child_with_error(FileNotFoundError(2, "No such file or directory: 'blah'")) - parent = MockService(backend=BACKEND, children={child.id: child}) - with self.service_patcher: + child = self.make_new_child_with_error(FileNotFoundError(2, "No such file or directory: 'blah'")) + parent = MockService(backend=BACKEND, children={child.id: child}) + child.serve() subscription, _ = parent.ask(service_id=child.id, input_values={}) @@ -229,10 +228,10 @@ def test_unknown_exceptions_in_responder_are_handled_and_sent_to_asker(self): class AnUnknownException(Exception): pass - child = self.make_new_child_with_error(AnUnknownException("This is an exception unknown to the asker.")) - parent = MockService(backend=BACKEND, children={child.id: child}) - with self.service_patcher: + child = self.make_new_child_with_error(AnUnknownException("This is an exception unknown to the asker.")) + parent = MockService(backend=BACKEND, children={child.id: child}) + child.serve() subscription, _ = parent.ask(service_id=child.id, input_values={}) @@ -247,11 +246,15 @@ def test_ask_with_real_run_function_with_no_log_message_forwarding(self): run function rather than a mock so that the underlying `Runner` instance is used, and check that remote log messages aren't forwarded to the local logger. """ - child = MockService(backend=BACKEND, service_id="truly/madly:deeply", run_function=self.create_run_function()) - parent = MockService(backend=BACKEND, children={child.id: child}) + with self.service_patcher: + child = MockService( + backend=BACKEND, + service_id="truly/madly:deeply", + run_function=self.create_run_function(), + ) + parent = MockService(backend=BACKEND, children={child.id: child}) - with self.assertLogs() as logging_context: - with self.service_patcher: + with self.assertLogs() as logging_context: child.serve() subscription, _ = parent.ask(service_id=child.id, input_values={}, subscribe_to_logs=False) answer = parent.wait_for_answer(subscription) @@ -268,16 +271,16 @@ def test_ask_with_real_run_function_with_log_message_forwarding(self): run function rather than a mock so that the underlying `Runner` instance is used, and check that remote log messages are forwarded to the local logger. """ - child = MockService( - backend=BACKEND, - service_id="my-super/service:6.0.1", - run_function=self.create_run_function(), - ) + with self.service_patcher: + child = MockService( + backend=BACKEND, + service_id="my-super/service:6.0.1", + run_function=self.create_run_function(), + ) - parent = MockService(backend=BACKEND, children={child.id: child}) + parent = MockService(backend=BACKEND, children={child.id: child}) - with self.assertLogs() as logs_context_manager: - with self.service_patcher: + with self.assertLogs() as logs_context_manager: child.serve() subscription, _ = parent.ask(service_id=child.id, input_values={}, subscribe_to_logs=True) answer = parent.wait_for_answer(subscription) @@ -319,14 +322,14 @@ def mock_app(analysis): return Runner(app_src=mock_app, twine='{"input_values_schema": {"type": "object", "required": []}}').run - child = MockService(backend=BACKEND, run_function=create_exception_logging_run_function()) - parent = MockService(backend=BACKEND, children={child.id: child}) + with self.service_patcher: + child = MockService(backend=BACKEND, run_function=create_exception_logging_run_function()) + parent = MockService(backend=BACKEND, children={child.id: child}) - with self.assertLogs(level=logging.ERROR) as logs_context_manager: - with self.service_patcher: + with self.assertLogs(level=logging.ERROR) as logs_context_manager: child.serve() subscription, _ = parent.ask(service_id=child.id, input_values={}, subscribe_to_logs=True) - parent.wait_for_answer(subscription, timeout=10) + parent.wait_for_answer(subscription, timeout=100) error_logged = False @@ -362,10 +365,10 @@ def mock_app(analysis): return Runner(app_src=mock_app, twine=twine).run - child = MockService(backend=BACKEND, run_function=create_run_function_with_monitoring()) - parent = MockService(backend=BACKEND, children={child.id: child}) - with self.service_patcher: + child = MockService(backend=BACKEND, run_function=create_run_function_with_monitoring()) + parent = MockService(backend=BACKEND, children={child.id: child}) + child.serve() subscription, _ = parent.ask(child.id, input_values={}) @@ -401,10 +404,10 @@ def mock_app(analysis): return Runner(app_src=mock_app, twine=twine).run - child = MockService(backend=BACKEND, run_function=create_run_function_with_monitoring()) - parent = MockService(backend=BACKEND, children={child.id: child}) - with self.service_patcher: + child = MockService(backend=BACKEND, run_function=create_run_function_with_monitoring()) + parent = MockService(backend=BACKEND, children={child.id: child}) + child.serve() subscription, _ = parent.ask(child.id, input_values={}) monitoring_data = [] @@ -421,16 +424,14 @@ def test_ask_with_non_json_python_primitive_input_values(self): """Test that non-JSON python primitive values (in this case a set and a datetime) can be sent and received by services. """ + input_values = {"my_set": {1, 2, 3}, "my_datetime": datetime.datetime.now()} def run_function(analysis_id, input_values, *args, **kwargs): return MockAnalysis(output_values=input_values) - child = MockService(backend=BACKEND, run_function=lambda *args, **kwargs: run_function(*args, **kwargs)) - parent = MockService(backend=BACKEND, children={child.id: child}) - - input_values = {"my_set": {1, 2, 3}, "my_datetime": datetime.datetime.now()} - with self.service_patcher: + child = MockService(backend=BACKEND, run_function=lambda *args, **kwargs: run_function(*args, **kwargs)) + parent = MockService(backend=BACKEND, children={child.id: child}) child.serve() subscription, _ = parent.ask( @@ -448,9 +449,6 @@ def test_ask_with_input_manifest(self): """Test that a service can ask a question including an input manifest to another service that is serving and receive an answer. """ - child = self.make_new_child(BACKEND, run_function_returnee=MockAnalysis()) - parent = MockService(backend=BACKEND, children={child.id: child}) - dataset_path = f"gs://{TEST_BUCKET_NAME}/my-dataset" input_manifest = Manifest( @@ -463,6 +461,8 @@ def test_ask_with_input_manifest(self): ) with self.service_patcher: + child = self.make_new_child(BACKEND, run_function_returnee=MockAnalysis()) + parent = MockService(backend=BACKEND, children={child.id: child}) child.serve() with patch("google.cloud.storage.blob.Blob.generate_signed_url", new=mock_generate_signed_url): @@ -478,9 +478,6 @@ def test_ask_with_input_manifest_and_no_input_values(self): """Test that a service can ask a question including an input manifest and no input values to another service that is serving and receive an answer. """ - child = self.make_new_child(BACKEND, run_function_returnee=MockAnalysis()) - parent = MockService(backend=BACKEND, children={child.id: child}) - dataset_path = f"gs://{TEST_BUCKET_NAME}/my-dataset" input_manifest = Manifest( @@ -493,6 +490,8 @@ def test_ask_with_input_manifest_and_no_input_values(self): ) with self.service_patcher: + child = self.make_new_child(BACKEND, run_function_returnee=MockAnalysis()) + parent = MockService(backend=BACKEND, children={child.id: child}) child.serve() with patch("google.cloud.storage.blob.Blob.generate_signed_url", new=mock_generate_signed_url): @@ -508,7 +507,8 @@ def test_ask_with_input_manifest_with_local_paths_raises_error(self): """Test that an error is raised if an input manifest whose datasets and/or files are not located in the cloud is used in a question. """ - service = MockService(backend=BACKEND) + with self.service_patcher: + service = MockService(backend=BACKEND) with self.assertRaises(exceptions.FileLocationError): service.ask( @@ -536,10 +536,9 @@ def run_function(*args, **kwargs): with open(temporary_local_path) as f: return MockAnalysis(output_values=f.read()) - child = MockService(backend=BACKEND, run_function=run_function) - parent = MockService(backend=BACKEND, children={child.id: child}) - with self.service_patcher: + child = MockService(backend=BACKEND, run_function=run_function) + parent = MockService(backend=BACKEND, children={child.id: child}) child.serve() subscription, _ = parent.ask( @@ -555,44 +554,42 @@ def run_function(*args, **kwargs): def test_ask_with_output_manifest(self): """Test that a service can receive an output manifest as part of the answer to a question.""" - child = self.make_new_child(BACKEND, run_function_returnee=MockAnalysisWithOutputManifest()) - parent = MockService(backend=BACKEND, children={child.id: child}) - with self.service_patcher: + child = self.make_new_child(BACKEND, run_function_returnee=MockAnalysisWithOutputManifest()) + parent = MockService(backend=BACKEND, children={child.id: child}) child.serve() + subscription, _ = parent.ask(service_id=child.id, input_values={}) answer = parent.wait_for_answer(subscription) self.assertEqual(answer["output_values"], MockAnalysisWithOutputManifest.output_values) self.assertEqual(answer["output_manifest"].id, MockAnalysisWithOutputManifest.output_manifest.id) - def test_service_can_ask_multiple_questions_to_child(self): - """Test that a service can ask multiple questions to the same child and expect replies to them all.""" - child = self.make_new_child(BACKEND, run_function_returnee=MockAnalysis()) - parent = MockService(backend=BACKEND, children={child.id: child}) - - with self.service_patcher: - child.serve() - answers = [] - - for i in range(5): - subscription, _ = parent.ask(service_id=child.id, input_values={}) - answers.append(parent.wait_for_answer(subscription)) - - for answer in answers: - self.assertEqual( - answer, - {"output_values": MockAnalysis().output_values, "output_manifest": MockAnalysis().output_manifest}, - ) + # def test_service_can_ask_multiple_questions_to_child(self): + # """Test that a service can ask multiple questions to the same child and expect replies to them all.""" + # with self.service_patcher: + # child = self.make_new_child(BACKEND, run_function_returnee=MockAnalysis()) + # parent = MockService(backend=BACKEND, children={child.id: child}) + # child.serve() + # answers = [] + # + # for i in range(5): + # subscription, _ = parent.ask(service_id=child.id, input_values={}) + # answers.append(parent.wait_for_answer(subscription)) + # + # for answer in answers: + # self.assertEqual( + # answer, + # {"output_values": MockAnalysis().output_values, "output_manifest": MockAnalysis().output_manifest}, + # ) def test_service_can_ask_questions_to_multiple_children(self): """Test that a service can ask questions to different children and expect replies to them all.""" - child_1 = self.make_new_child(BACKEND, run_function_returnee=MockAnalysis()) - child_2 = self.make_new_child(BACKEND, run_function_returnee=DifferentMockAnalysis()) - - parent = MockService(backend=BACKEND, children={child_1.id: child_1, child_2.id: child_2}) - with self.service_patcher: + child_1 = self.make_new_child(BACKEND, run_function_returnee=MockAnalysis()) + child_2 = self.make_new_child(BACKEND, run_function_returnee=DifferentMockAnalysis()) + parent = MockService(backend=BACKEND, children={child_1.id: child_1, child_2.id: child_2}) + child_1.serve() child_2.serve() @@ -615,106 +612,108 @@ def test_service_can_ask_questions_to_multiple_children(self): }, ) - def test_child_can_ask_its_own_child_questions(self): - """Test that a child can contact its own child while answering a question from a parent.""" - child_of_child = self.make_new_child(BACKEND, run_function_returnee=DifferentMockAnalysis()) - - def child_run_function(analysis_id, input_values, *args, **kwargs): - subscription, _ = child.ask(service_id=child_of_child.id, input_values=input_values) - return MockAnalysis(output_values={input_values["question"]: child.wait_for_answer(subscription)}) - - child = MockService( - backend=BACKEND, - run_function=child_run_function, - children={child_of_child.id: child_of_child}, - ) - - parent = MockService(backend=BACKEND, children={child.id: child}) - - with self.service_patcher: - child.serve() - child_of_child.serve() - - subscription, _ = parent.ask( - service_id=child.id, - input_values={"question": "What does the child of the child say?"}, - ) - - answer = parent.wait_for_answer(subscription) - - self.assertEqual( - answer, - { - "output_values": { - "What does the child of the child say?": { - "output_values": DifferentMockAnalysis.output_values, - "output_manifest": DifferentMockAnalysis.output_manifest, - } - }, - "output_manifest": None, - }, - ) - - def test_child_can_ask_its_own_children_questions(self): - """Test that a child can contact more than one of its own children while answering a question from a parent.""" - first_child_of_child = self.make_new_child(BACKEND, run_function_returnee=DifferentMockAnalysis()) - second_child_of_child = self.make_new_child(BACKEND, run_function_returnee=MockAnalysis()) - - def child_run_function(analysis_id, input_values, *args, **kwargs): - subscription_1, _ = child.ask(service_id=first_child_of_child.id, input_values=input_values) - subscription_2, _ = child.ask(service_id=second_child_of_child.id, input_values=input_values) - - return MockAnalysis( - output_values={ - "first_child_of_child": child.wait_for_answer(subscription_1), - "second_child_of_child": child.wait_for_answer(subscription_2), - } - ) - - child = MockService( - backend=BACKEND, - run_function=child_run_function, - children={first_child_of_child.id: first_child_of_child, second_child_of_child.id: second_child_of_child}, - ) - - parent = MockService(backend=BACKEND, children={child.id: child}) - - with self.service_patcher: - child.serve() - first_child_of_child.serve() - second_child_of_child.serve() - - subscription, _ = parent.ask( - service_id=child.id, - input_values={"question": "What does the child of the child say?"}, - ) - - answer = parent.wait_for_answer(subscription) - - self.assertEqual( - answer, - { - "output_values": { - "first_child_of_child": { - "output_values": DifferentMockAnalysis.output_values, - "output_manifest": DifferentMockAnalysis.output_manifest, - }, - "second_child_of_child": { - "output_values": MockAnalysis().output_values, - "output_manifest": MockAnalysis().output_manifest, - }, - }, - "output_manifest": None, - }, - ) + # def test_child_can_ask_its_own_child_questions(self): + # """Test that a child can contact its own child while answering a question from a parent.""" + # def child_run_function(analysis_id, input_values, *args, **kwargs): + # subscription, _ = child.ask(service_id=child_of_child.id, input_values=input_values) + # return MockAnalysis(output_values={input_values["question"]: child.wait_for_answer(subscription)}) + # + # with self.service_patcher: + # child_of_child = self.make_new_child(BACKEND, run_function_returnee=DifferentMockAnalysis()) + # + # child = MockService( + # backend=BACKEND, + # run_function=child_run_function, + # children={child_of_child.id: child_of_child}, + # ) + # + # parent = MockService(backend=BACKEND, children={child.id: child}) + # + # child.serve() + # child_of_child.serve() + # + # subscription, _ = parent.ask( + # service_id=child.id, + # input_values={"question": "What does the child of the child say?"}, + # ) + # + # answer = parent.wait_for_answer(subscription) + # + # self.assertEqual( + # answer, + # { + # "output_values": { + # "What does the child of the child say?": { + # "output_values": DifferentMockAnalysis.output_values, + # "output_manifest": DifferentMockAnalysis.output_manifest, + # } + # }, + # "output_manifest": None, + # }, + # ) + + # def test_child_can_ask_its_own_children_questions(self): + # """Test that a child can contact more than one of its own children while answering a question from a parent.""" + # + # def child_run_function(analysis_id, input_values, *args, **kwargs): + # subscription_1, _ = child.ask(service_id=first_child_of_child.id, input_values=input_values) + # subscription_2, _ = child.ask(service_id=second_child_of_child.id, input_values=input_values) + # + # return MockAnalysis( + # output_values={ + # "first_child_of_child": child.wait_for_answer(subscription_1), + # "second_child_of_child": child.wait_for_answer(subscription_2), + # } + # ) + # + # with self.service_patcher: + # first_child_of_child = self.make_new_child(BACKEND, run_function_returnee=DifferentMockAnalysis()) + # second_child_of_child = self.make_new_child(BACKEND, run_function_returnee=MockAnalysis()) + # + # child = MockService( + # backend=BACKEND, + # run_function=child_run_function, + # children={first_child_of_child.id: first_child_of_child, + # second_child_of_child.id: second_child_of_child}, + # ) + # + # parent = MockService(backend=BACKEND, children={child.id: child}) + # + # child.serve() + # first_child_of_child.serve() + # second_child_of_child.serve() + # + # subscription, _ = parent.ask( + # service_id=child.id, + # input_values={"question": "What does the child of the child say?"}, + # ) + # + # answer = parent.wait_for_answer(subscription) + # + # self.assertEqual( + # answer, + # { + # "output_values": { + # "first_child_of_child": { + # "output_values": DifferentMockAnalysis.output_values, + # "output_manifest": DifferentMockAnalysis.output_manifest, + # }, + # "second_child_of_child": { + # "output_values": MockAnalysis().output_values, + # "output_manifest": MockAnalysis().output_manifest, + # }, + # }, + # "output_manifest": None, + # }, + # ) def test_child_messages_can_be_recorded_by_parent(self): """Test that the parent can record messages it receives from its child to a JSON file.""" - child = MockService(backend=BACKEND, run_function=self.create_run_function()) - parent = MockService(backend=BACKEND, children={child.id: child}) - with self.service_patcher: + child = MockService(backend=BACKEND, run_function=self.create_run_function()) + parent = MockService(backend=BACKEND, children={child.id: child}) child.serve() + subscription, _ = parent.ask(service_id=child.id, input_values={}, subscribe_to_logs=True) parent.wait_for_answer(subscription) @@ -727,10 +726,9 @@ def test_child_messages_can_be_recorded_by_parent(self): def test_child_exception_message_can_be_recorded_by_parent(self): """Test that the parent can record exceptions raised by the child.""" - child = self.make_new_child_with_error(ValueError("Oh no.")) - parent = MockService(backend=BACKEND, children={child.id: child}) - with self.service_patcher: + child = self.make_new_child_with_error(ValueError("Oh no.")) + parent = MockService(backend=BACKEND, children={child.id: child}) child.serve() with self.assertRaises(ValueError): @@ -750,10 +748,9 @@ def run_function(*args, **kwargs): time.sleep(0.3) return MockAnalysis() - child = MockService(backend=BACKEND, run_function=lambda *args, **kwargs: run_function()) - parent = MockService(backend=BACKEND, children={child.id: child}) - with self.service_patcher: + child = MockService(backend=BACKEND, run_function=lambda *args, **kwargs: run_function()) + parent = MockService(backend=BACKEND, children={child.id: child}) child.serve() with patch( @@ -797,10 +794,9 @@ def run_function(*args, **kwargs): analysis.output_values = {"tada": True} return analysis - child = MockService(backend=BACKEND, run_function=run_function) - parent = MockService(backend=BACKEND, children={child.id: child}) - with self.service_patcher: + child = MockService(backend=BACKEND, run_function=run_function) + parent = MockService(backend=BACKEND, children={child.id: child}) child.serve() subscription, _ = parent.ask( @@ -823,79 +819,79 @@ def run_function(*args, **kwargs): # Check that the result was received correctly. self.assertEqual(result["output_values"], {"tada": True}) - def test_providing_dynamic_children(self): - """Test that, if children are provided to the `ask` method while asking a question, the child being asked uses - those children instead of the children it has defined in its app configuration. - """ - - def mock_child_app(analysis): - analysis.children["expected_child"]._service = child - analysis.output_values = analysis.children["expected_child"].ask(input_values=[1, 2, 3, 4])["output_values"] - - static_children = [ - { - "key": "expected_child", - "id": f"octue/static-child-of-child:{MOCK_SERVICE_REVISION_TAG}", - "backend": {"name": "GCPPubSubBackend", "project_name": "my-project"}, - }, - ] - - runner = Runner( - app_src=mock_child_app, - twine={ - "children": [{"key": "expected_child"}], - "input_values_schema": {"type": "object", "required": []}, - "output_values_schema": {}, - }, - children=static_children, - service_id=f"octue/child:{MOCK_SERVICE_REVISION_TAG}", - ) - - static_child_of_child = self.make_new_child( - backend=BACKEND, - service_id=f"octue/static-child-of-child:{MOCK_SERVICE_REVISION_TAG}", - run_function_returnee=MockAnalysis(output_values="I am the static child."), - ) - - dynamic_child_of_child = self.make_new_child( - backend=BACKEND, - service_id=f"octue/dynamic-child-of-child:{MOCK_SERVICE_REVISION_TAG}", - run_function_returnee=MockAnalysis(output_values="I am the dynamic child."), - ) - - child = MockService( - backend=BACKEND, - service_id=f"octue/child:{MOCK_SERVICE_REVISION_TAG}", - run_function=runner.run, - children={ - static_child_of_child.id: static_child_of_child, - dynamic_child_of_child.id: dynamic_child_of_child, - }, - ) - - parent = MockService( - backend=BACKEND, - service_id=f"octue/parent:{MOCK_SERVICE_REVISION_TAG}", - children={child.id: child}, - ) - - dynamic_children = [ - { - "key": "expected_child", - "id": f"octue/dynamic-child-of-child:{MOCK_SERVICE_REVISION_TAG}", - "backend": {"name": "GCPPubSubBackend", "project_name": "my-project"}, - }, - ] - - with self.service_patcher: - static_child_of_child.serve() - dynamic_child_of_child.serve() - child.serve() - - subscription, _ = parent.ask(service_id=child.id, input_values={}, children=dynamic_children) - answer = parent.wait_for_answer(subscription) - - self.assertEqual(answer["output_values"], "I am the dynamic child.") + # def test_providing_dynamic_children(self): + # """Test that, if children are provided to the `ask` method while asking a question, the child being asked uses + # those children instead of the children it has defined in its app configuration. + # """ + # + # def mock_child_app(analysis): + # analysis.children["expected_child"]._service = child + # analysis.output_values = analysis.children["expected_child"].ask(input_values=[1, 2, 3, 4])["output_values"] + # + # static_children = [ + # { + # "key": "expected_child", + # "id": f"octue/static-child-of-child:{MOCK_SERVICE_REVISION_TAG}", + # "backend": {"name": "GCPPubSubBackend", "project_name": "my-project"}, + # }, + # ] + # + # runner = Runner( + # app_src=mock_child_app, + # twine={ + # "children": [{"key": "expected_child"}], + # "input_values_schema": {"type": "object", "required": []}, + # "output_values_schema": {}, + # }, + # children=static_children, + # service_id=f"octue/child:{MOCK_SERVICE_REVISION_TAG}", + # ) + # + # with self.service_patcher: + # static_child_of_child = self.make_new_child( + # backend=BACKEND, + # service_id=f"octue/static-child-of-child:{MOCK_SERVICE_REVISION_TAG}", + # run_function_returnee=MockAnalysis(output_values="I am the static child."), + # ) + # + # dynamic_child_of_child = self.make_new_child( + # backend=BACKEND, + # service_id=f"octue/dynamic-child-of-child:{MOCK_SERVICE_REVISION_TAG}", + # run_function_returnee=MockAnalysis(output_values="I am the dynamic child."), + # ) + # + # child = MockService( + # backend=BACKEND, + # service_id=f"octue/child:{MOCK_SERVICE_REVISION_TAG}", + # run_function=runner.run, + # children={ + # static_child_of_child.id: static_child_of_child, + # dynamic_child_of_child.id: dynamic_child_of_child, + # }, + # ) + # + # parent = MockService( + # backend=BACKEND, + # service_id=f"octue/parent:{MOCK_SERVICE_REVISION_TAG}", + # children={child.id: child}, + # ) + # + # static_child_of_child.serve() + # dynamic_child_of_child.serve() + # child.serve() + # + # dynamic_children = [ + # { + # "key": "expected_child", + # "id": f"octue/dynamic-child-of-child:{MOCK_SERVICE_REVISION_TAG}", + # "backend": {"name": "GCPPubSubBackend", "project_name": "my-project"}, + # }, + # ] + # + # subscription, _ = parent.ask(service_id=child.id, input_values={}, children=dynamic_children) + # answer = parent.wait_for_answer(subscription) + # + # self.assertEqual(answer["output_values"], "I am the dynamic child.") @staticmethod def make_new_child(backend, run_function_returnee, service_id=None): From df62601603f35e5b85bd6f1f24af6b198c30d998 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 2 Apr 2024 18:15:55 +0100 Subject: [PATCH 095/169] FIX: Fix event order --- octue/cloud/pub_sub/service.py | 41 +++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index f3b35ce6f..6dea282fb 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -93,11 +93,6 @@ def __init__(self, backend, service_id=None, run_function=None, name=None, servi self._publisher = None self._event_handler = None - self.topic = Topic(name=self.backend.services_namespace, project_name=self.backend.project_name) - - if not self.topic.exists(): - raise octue.exceptions.ServiceNotFound(f"Topic {self.backend.services_namespace!r} cannot be found.") - def __repr__(self): """Represent the service as a string. @@ -144,7 +139,7 @@ def serve(self, timeout=None, delete_topic_and_subscription_on_exit=False, allow subscription = Subscription( name=".".join((self.backend.services_namespace, self._pub_sub_id)), - topic=self.topic, + topic=self._get_services_topic(), filter=f'attributes.recipient = "{self.id}" AND attributes.sender_type = "{PARENT_SENDER_TYPE}"', expiration_time=None, ) @@ -211,13 +206,15 @@ def answer(self, question, heartbeat_interval=120, timeout=30): heartbeater = None + services_topic = self._get_services_topic() + try: - self._send_delivery_acknowledgment(self.topic, question_uuid, originator) + self._send_delivery_acknowledgment(services_topic, question_uuid, originator) heartbeater = RepeatingTimer( interval=heartbeat_interval, function=self._send_heartbeat, - kwargs={"topic": self.topic, "question_uuid": question_uuid, "originator": originator}, + kwargs={"topic": services_topic, "question_uuid": question_uuid, "originator": originator}, ) heartbeater.daemon = True @@ -226,7 +223,7 @@ def answer(self, question, heartbeat_interval=120, timeout=30): if forward_logs: analysis_log_handler = GoogleCloudPubSubHandler( message_sender=self._send_message, - topic=self.topic, + topic=services_topic, question_uuid=question_uuid, originator=originator, recipient=originator, @@ -242,7 +239,7 @@ def answer(self, question, heartbeat_interval=120, timeout=30): analysis_log_handler=analysis_log_handler, handle_monitor_message=functools.partial( self._send_monitor_message, - topic=self.topic, + topic=services_topic, question_uuid=question_uuid, originator=originator, ), @@ -256,7 +253,7 @@ def answer(self, question, heartbeat_interval=120, timeout=30): self._send_message( message=result, - topic=self.topic, + topic=services_topic, originator=originator, recipient=originator, attributes={"question_uuid": question_uuid, "sender_type": CHILD_SENDER_TYPE}, @@ -271,7 +268,7 @@ def answer(self, question, heartbeat_interval=120, timeout=30): heartbeater.cancel() warn_if_incompatible(child_sdk_version=self._local_sdk_version, parent_sdk_version=parent_sdk_version) - self.send_exception(self.topic, question_uuid, originator, timeout=timeout) + self.send_exception(services_topic, question_uuid, originator, timeout=timeout) raise error def ask( @@ -331,15 +328,16 @@ def ask( ) question_uuid = question_uuid or str(uuid.uuid4()) + services_topic = self._get_services_topic() if asynchronous and not push_endpoint: answer_subscription = None else: - pub_sub_id = convert_service_id_to_pub_sub_form(service_id) + pub_sub_id = convert_service_id_to_pub_sub_form(self.id) answer_subscription = Subscription( - name=".".join((self.backend.services_namespace, pub_sub_id, ANSWERS_NAMESPACE, question_uuid)), - topic=self.topic, + name=".".join((self.backend.services_namespace, ANSWERS_NAMESPACE, pub_sub_id, question_uuid)), + topic=services_topic, filter=( f'attributes.sender = "{service_id}" ' f'attributes.recipient = "{self.id}" ' @@ -357,6 +355,7 @@ def ask( service_id=service_id, forward_logs=subscribe_to_logs, save_diagnostics=save_diagnostics, + topic=services_topic, question_uuid=question_uuid, ) @@ -430,6 +429,14 @@ def send_exception(self, topic, question_uuid, originator, timeout=30): timeout=timeout, ) + def _get_services_topic(self): + topic = Topic(name=self.backend.services_namespace, project_name=self.backend.project_name) + + if not topic.exists(): + raise octue.exceptions.ServiceNotFound(f"Topic {self.backend.services_namespace!r} cannot be found.") + + return topic + def _send_message(self, message, topic, originator, recipient, attributes=None, timeout=30): """Send a JSON-serialised message to the given topic with optional message attributes and increment the `messages_published` attribute of the topic by one. This method is thread-safe. @@ -481,6 +488,7 @@ def _send_question( service_id, forward_logs, save_diagnostics, + topic, question_uuid, timeout=30, ): @@ -492,6 +500,7 @@ def _send_question( :param str service_id: the ID of the child to send the question to :param bool forward_logs: whether to request the child to forward its logs :param str save_diagnostics: must be one of {"SAVE_DIAGNOSTICS_OFF", "SAVE_DIAGNOSTICS_ON_CRASH", "SAVE_DIAGNOSTICS_ON"}; if turned on, allow the input values and manifest (and its datasets) to be saved by the child either all the time or just if it fails while processing them + :param topic: :param str question_uuid: the UUID of the question being sent :param float timeout: time in seconds after which to give up sending :return None: @@ -504,7 +513,7 @@ def _send_question( future = self._send_message( message=question, - topic=self.topic, + topic=topic, timeout=timeout, originator=self.id, recipient=service_id, From e3a1b3f3edfeaaba7e557dab241b727eb2bd844f Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 2 Apr 2024 18:44:24 +0100 Subject: [PATCH 096/169] TST: Uncomment service tests --- tests/cloud/pub_sub/test_service.py | 371 ++++++++++++++-------------- 1 file changed, 187 insertions(+), 184 deletions(-) diff --git a/tests/cloud/pub_sub/test_service.py b/tests/cloud/pub_sub/test_service.py index 0a1c403bc..2133b1399 100644 --- a/tests/cloud/pub_sub/test_service.py +++ b/tests/cloud/pub_sub/test_service.py @@ -565,23 +565,23 @@ def test_ask_with_output_manifest(self): self.assertEqual(answer["output_values"], MockAnalysisWithOutputManifest.output_values) self.assertEqual(answer["output_manifest"].id, MockAnalysisWithOutputManifest.output_manifest.id) - # def test_service_can_ask_multiple_questions_to_child(self): - # """Test that a service can ask multiple questions to the same child and expect replies to them all.""" - # with self.service_patcher: - # child = self.make_new_child(BACKEND, run_function_returnee=MockAnalysis()) - # parent = MockService(backend=BACKEND, children={child.id: child}) - # child.serve() - # answers = [] - # - # for i in range(5): - # subscription, _ = parent.ask(service_id=child.id, input_values={}) - # answers.append(parent.wait_for_answer(subscription)) - # - # for answer in answers: - # self.assertEqual( - # answer, - # {"output_values": MockAnalysis().output_values, "output_manifest": MockAnalysis().output_manifest}, - # ) + def test_service_can_ask_multiple_questions_to_child(self): + """Test that a service can ask multiple questions to the same child and expect replies to them all.""" + with self.service_patcher: + child = self.make_new_child(BACKEND, run_function_returnee=MockAnalysis()) + parent = MockService(backend=BACKEND, children={child.id: child}) + child.serve() + answers = [] + + for i in range(5): + subscription, _ = parent.ask(service_id=child.id, input_values={}) + answers.append(parent.wait_for_answer(subscription, timeout=3600)) + + for answer in answers: + self.assertEqual( + answer, + {"output_values": MockAnalysis().output_values, "output_manifest": MockAnalysis().output_manifest}, + ) def test_service_can_ask_questions_to_multiple_children(self): """Test that a service can ask questions to different children and expect replies to them all.""" @@ -612,100 +612,103 @@ def test_service_can_ask_questions_to_multiple_children(self): }, ) - # def test_child_can_ask_its_own_child_questions(self): - # """Test that a child can contact its own child while answering a question from a parent.""" - # def child_run_function(analysis_id, input_values, *args, **kwargs): - # subscription, _ = child.ask(service_id=child_of_child.id, input_values=input_values) - # return MockAnalysis(output_values={input_values["question"]: child.wait_for_answer(subscription)}) - # - # with self.service_patcher: - # child_of_child = self.make_new_child(BACKEND, run_function_returnee=DifferentMockAnalysis()) - # - # child = MockService( - # backend=BACKEND, - # run_function=child_run_function, - # children={child_of_child.id: child_of_child}, - # ) - # - # parent = MockService(backend=BACKEND, children={child.id: child}) - # - # child.serve() - # child_of_child.serve() - # - # subscription, _ = parent.ask( - # service_id=child.id, - # input_values={"question": "What does the child of the child say?"}, - # ) - # - # answer = parent.wait_for_answer(subscription) - # - # self.assertEqual( - # answer, - # { - # "output_values": { - # "What does the child of the child say?": { - # "output_values": DifferentMockAnalysis.output_values, - # "output_manifest": DifferentMockAnalysis.output_manifest, - # } - # }, - # "output_manifest": None, - # }, - # ) - - # def test_child_can_ask_its_own_children_questions(self): - # """Test that a child can contact more than one of its own children while answering a question from a parent.""" - # - # def child_run_function(analysis_id, input_values, *args, **kwargs): - # subscription_1, _ = child.ask(service_id=first_child_of_child.id, input_values=input_values) - # subscription_2, _ = child.ask(service_id=second_child_of_child.id, input_values=input_values) - # - # return MockAnalysis( - # output_values={ - # "first_child_of_child": child.wait_for_answer(subscription_1), - # "second_child_of_child": child.wait_for_answer(subscription_2), - # } - # ) - # - # with self.service_patcher: - # first_child_of_child = self.make_new_child(BACKEND, run_function_returnee=DifferentMockAnalysis()) - # second_child_of_child = self.make_new_child(BACKEND, run_function_returnee=MockAnalysis()) - # - # child = MockService( - # backend=BACKEND, - # run_function=child_run_function, - # children={first_child_of_child.id: first_child_of_child, - # second_child_of_child.id: second_child_of_child}, - # ) - # - # parent = MockService(backend=BACKEND, children={child.id: child}) - # - # child.serve() - # first_child_of_child.serve() - # second_child_of_child.serve() - # - # subscription, _ = parent.ask( - # service_id=child.id, - # input_values={"question": "What does the child of the child say?"}, - # ) - # - # answer = parent.wait_for_answer(subscription) - # - # self.assertEqual( - # answer, - # { - # "output_values": { - # "first_child_of_child": { - # "output_values": DifferentMockAnalysis.output_values, - # "output_manifest": DifferentMockAnalysis.output_manifest, - # }, - # "second_child_of_child": { - # "output_values": MockAnalysis().output_values, - # "output_manifest": MockAnalysis().output_manifest, - # }, - # }, - # "output_manifest": None, - # }, - # ) + def test_child_can_ask_its_own_child_questions(self): + """Test that a child can contact its own child while answering a question from a parent.""" + + def child_run_function(analysis_id, input_values, *args, **kwargs): + subscription, _ = child.ask(service_id=child_of_child.id, input_values=input_values) + return MockAnalysis(output_values={input_values["question"]: child.wait_for_answer(subscription)}) + + with self.service_patcher: + child_of_child = self.make_new_child(BACKEND, run_function_returnee=DifferentMockAnalysis()) + + child = MockService( + backend=BACKEND, + run_function=child_run_function, + children={child_of_child.id: child_of_child}, + ) + + parent = MockService(backend=BACKEND, children={child.id: child}) + + child.serve() + child_of_child.serve() + + subscription, _ = parent.ask( + service_id=child.id, + input_values={"question": "What does the child of the child say?"}, + ) + + answer = parent.wait_for_answer(subscription) + + self.assertEqual( + answer, + { + "output_values": { + "What does the child of the child say?": { + "output_values": DifferentMockAnalysis.output_values, + "output_manifest": DifferentMockAnalysis.output_manifest, + } + }, + "output_manifest": None, + }, + ) + + def test_child_can_ask_its_own_children_questions(self): + """Test that a child can contact more than one of its own children while answering a question from a parent.""" + + def child_run_function(analysis_id, input_values, *args, **kwargs): + subscription_1, _ = child.ask(service_id=first_child_of_child.id, input_values=input_values) + subscription_2, _ = child.ask(service_id=second_child_of_child.id, input_values=input_values) + + return MockAnalysis( + output_values={ + "first_child_of_child": child.wait_for_answer(subscription_1), + "second_child_of_child": child.wait_for_answer(subscription_2), + } + ) + + with self.service_patcher: + first_child_of_child = self.make_new_child(BACKEND, run_function_returnee=DifferentMockAnalysis()) + second_child_of_child = self.make_new_child(BACKEND, run_function_returnee=MockAnalysis()) + + child = MockService( + backend=BACKEND, + run_function=child_run_function, + children={ + first_child_of_child.id: first_child_of_child, + second_child_of_child.id: second_child_of_child, + }, + ) + + parent = MockService(backend=BACKEND, children={child.id: child}) + + child.serve() + first_child_of_child.serve() + second_child_of_child.serve() + + subscription, _ = parent.ask( + service_id=child.id, + input_values={"question": "What does the child of the child say?"}, + ) + + answer = parent.wait_for_answer(subscription) + + self.assertEqual( + answer, + { + "output_values": { + "first_child_of_child": { + "output_values": DifferentMockAnalysis.output_values, + "output_manifest": DifferentMockAnalysis.output_manifest, + }, + "second_child_of_child": { + "output_values": MockAnalysis().output_values, + "output_manifest": MockAnalysis().output_manifest, + }, + }, + "output_manifest": None, + }, + ) def test_child_messages_can_be_recorded_by_parent(self): """Test that the parent can record messages it receives from its child to a JSON file.""" @@ -819,79 +822,79 @@ def run_function(*args, **kwargs): # Check that the result was received correctly. self.assertEqual(result["output_values"], {"tada": True}) - # def test_providing_dynamic_children(self): - # """Test that, if children are provided to the `ask` method while asking a question, the child being asked uses - # those children instead of the children it has defined in its app configuration. - # """ - # - # def mock_child_app(analysis): - # analysis.children["expected_child"]._service = child - # analysis.output_values = analysis.children["expected_child"].ask(input_values=[1, 2, 3, 4])["output_values"] - # - # static_children = [ - # { - # "key": "expected_child", - # "id": f"octue/static-child-of-child:{MOCK_SERVICE_REVISION_TAG}", - # "backend": {"name": "GCPPubSubBackend", "project_name": "my-project"}, - # }, - # ] - # - # runner = Runner( - # app_src=mock_child_app, - # twine={ - # "children": [{"key": "expected_child"}], - # "input_values_schema": {"type": "object", "required": []}, - # "output_values_schema": {}, - # }, - # children=static_children, - # service_id=f"octue/child:{MOCK_SERVICE_REVISION_TAG}", - # ) - # - # with self.service_patcher: - # static_child_of_child = self.make_new_child( - # backend=BACKEND, - # service_id=f"octue/static-child-of-child:{MOCK_SERVICE_REVISION_TAG}", - # run_function_returnee=MockAnalysis(output_values="I am the static child."), - # ) - # - # dynamic_child_of_child = self.make_new_child( - # backend=BACKEND, - # service_id=f"octue/dynamic-child-of-child:{MOCK_SERVICE_REVISION_TAG}", - # run_function_returnee=MockAnalysis(output_values="I am the dynamic child."), - # ) - # - # child = MockService( - # backend=BACKEND, - # service_id=f"octue/child:{MOCK_SERVICE_REVISION_TAG}", - # run_function=runner.run, - # children={ - # static_child_of_child.id: static_child_of_child, - # dynamic_child_of_child.id: dynamic_child_of_child, - # }, - # ) - # - # parent = MockService( - # backend=BACKEND, - # service_id=f"octue/parent:{MOCK_SERVICE_REVISION_TAG}", - # children={child.id: child}, - # ) - # - # static_child_of_child.serve() - # dynamic_child_of_child.serve() - # child.serve() - # - # dynamic_children = [ - # { - # "key": "expected_child", - # "id": f"octue/dynamic-child-of-child:{MOCK_SERVICE_REVISION_TAG}", - # "backend": {"name": "GCPPubSubBackend", "project_name": "my-project"}, - # }, - # ] - # - # subscription, _ = parent.ask(service_id=child.id, input_values={}, children=dynamic_children) - # answer = parent.wait_for_answer(subscription) - # - # self.assertEqual(answer["output_values"], "I am the dynamic child.") + def test_providing_dynamic_children(self): + """Test that, if children are provided to the `ask` method while asking a question, the child being asked uses + those children instead of the children it has defined in its app configuration. + """ + + def mock_child_app(analysis): + analysis.children["expected_child"]._service = child + analysis.output_values = analysis.children["expected_child"].ask(input_values=[1, 2, 3, 4])["output_values"] + + static_children = [ + { + "key": "expected_child", + "id": f"octue/static-child-of-child:{MOCK_SERVICE_REVISION_TAG}", + "backend": {"name": "GCPPubSubBackend", "project_name": "my-project"}, + }, + ] + + runner = Runner( + app_src=mock_child_app, + twine={ + "children": [{"key": "expected_child"}], + "input_values_schema": {"type": "object", "required": []}, + "output_values_schema": {}, + }, + children=static_children, + service_id=f"octue/child:{MOCK_SERVICE_REVISION_TAG}", + ) + + with self.service_patcher: + static_child_of_child = self.make_new_child( + backend=BACKEND, + service_id=f"octue/static-child-of-child:{MOCK_SERVICE_REVISION_TAG}", + run_function_returnee=MockAnalysis(output_values="I am the static child."), + ) + + dynamic_child_of_child = self.make_new_child( + backend=BACKEND, + service_id=f"octue/dynamic-child-of-child:{MOCK_SERVICE_REVISION_TAG}", + run_function_returnee=MockAnalysis(output_values="I am the dynamic child."), + ) + + child = MockService( + backend=BACKEND, + service_id=f"octue/child:{MOCK_SERVICE_REVISION_TAG}", + run_function=runner.run, + children={ + static_child_of_child.id: static_child_of_child, + dynamic_child_of_child.id: dynamic_child_of_child, + }, + ) + + parent = MockService( + backend=BACKEND, + service_id=f"octue/parent:{MOCK_SERVICE_REVISION_TAG}", + children={child.id: child}, + ) + + static_child_of_child.serve() + dynamic_child_of_child.serve() + child.serve() + + dynamic_children = [ + { + "key": "expected_child", + "id": f"octue/dynamic-child-of-child:{MOCK_SERVICE_REVISION_TAG}", + "backend": {"name": "GCPPubSubBackend", "project_name": "my-project"}, + }, + ] + + subscription, _ = parent.ask(service_id=child.id, input_values={}, children=dynamic_children) + answer = parent.wait_for_answer(subscription) + + self.assertEqual(answer["output_values"], "I am the dynamic child.") @staticmethod def make_new_child(backend, run_function_returnee, service_id=None): From 414a75f4409181e1b260d4d3f375ae03072303fc Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 2 Apr 2024 18:53:01 +0100 Subject: [PATCH 097/169] OPS: Add `octue.services` topic --- terraform/pub_sub.tf | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 terraform/pub_sub.tf diff --git a/terraform/pub_sub.tf b/terraform/pub_sub.tf new file mode 100644 index 000000000..ce436afec --- /dev/null +++ b/terraform/pub_sub.tf @@ -0,0 +1,3 @@ +resource "google_pubsub_topic" "services_topic" { + name = "octue.services" +} From a572b541fb08d66b9dbd887081f95ad1cf1bf791 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 2 Apr 2024 18:58:16 +0100 Subject: [PATCH 098/169] TST: Update more tests --- tests/data/events.json | 64 ++++++++++++++++++++++++++---------------- tests/test_cli.py | 2 +- 2 files changed, 41 insertions(+), 25 deletions(-) diff --git a/tests/data/events.json b/tests/data/events.json index ec42d1409..4cd741a59 100644 --- a/tests/data/events.json +++ b/tests/data/events.json @@ -5,11 +5,13 @@ "kind": "delivery_acknowledgement" }, "attributes": { - "ordering_key": "0", + "order": "0", "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", + "originator": "octue/test-service:1.0.0", + "sender": "octue/test-service:1.0.0", "sender_type": "CHILD", - "version": "0.51.0", - "sender": "octue/test-service:1.0.0" + "sender_sdk_version": "0.51.0", + "recipient": "octue/another-service:3.2.1" } }, { @@ -39,11 +41,13 @@ } }, "attributes": { - "ordering_key": "1", + "order": "1", "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", + "originator": "octue/test-service:1.0.0", + "sender": "octue/test-service:1.0.0", "sender_type": "CHILD", - "version": "0.51.0", - "sender": "octue/test-service:1.0.0" + "sender_sdk_version": "0.51.0", + "recipient": "octue/another-service:3.2.1" } }, { @@ -73,11 +77,13 @@ } }, "attributes": { - "ordering_key": "2", + "order": "2", "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", + "originator": "octue/test-service:1.0.0", + "sender": "octue/test-service:1.0.0", "sender_type": "CHILD", - "version": "0.51.0", - "sender": "octue/test-service:1.0.0" + "sender_sdk_version": "0.51.0", + "recipient": "octue/another-service:3.2.1" } }, { @@ -107,11 +113,13 @@ } }, "attributes": { - "ordering_key": "3", + "order": "3", "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", + "originator": "octue/test-service:1.0.0", + "sender": "octue/test-service:1.0.0", "sender_type": "CHILD", - "version": "0.51.0", - "sender": "octue/test-service:1.0.0" + "sender_sdk_version": "0.51.0", + "recipient": "octue/another-service:3.2.1" } }, { @@ -141,11 +149,13 @@ } }, "attributes": { - "ordering_key": "4", + "order": "4", "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", + "originator": "octue/test-service:1.0.0", + "sender": "octue/test-service:1.0.0", "sender_type": "CHILD", - "version": "0.51.0", - "sender": "octue/test-service:1.0.0" + "sender_sdk_version": "0.51.0", + "recipient": "octue/another-service:3.2.1" } }, { @@ -175,11 +185,13 @@ } }, "attributes": { - "ordering_key": "5", + "order": "5", "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", + "originator": "octue/test-service:1.0.0", + "sender": "octue/test-service:1.0.0", "sender_type": "CHILD", - "version": "0.51.0", - "sender": "octue/test-service:1.0.0" + "sender_sdk_version": "0.51.0", + "recipient": "octue/another-service:3.2.1" } }, { @@ -204,11 +216,13 @@ "output_values": [1, 2, 3, 4, 5] }, "attributes": { - "ordering_key": "6", + "order": "6", "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", + "originator": "octue/test-service:1.0.0", + "sender": "octue/test-service:1.0.0", "sender_type": "CHILD", - "version": "0.51.0", - "sender": "octue/test-service:1.0.0" + "sender_sdk_version": "0.51.0", + "recipient": "octue/another-service:3.2.1" } }, { @@ -217,11 +231,13 @@ "kind": "heartbeat" }, "attributes": { - "ordering_key": "7", + "order": "7", "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", + "originator": "octue/test-service:1.0.0", + "sender": "octue/test-service:1.0.0", "sender_type": "CHILD", - "version": "0.51.0", - "sender": "octue/test-service:1.0.0" + "sender_sdk_version": "0.51.0", + "recipient": "octue/another-service:3.2.1" } } ] diff --git a/tests/test_cli.py b/tests/test_cli.py index 7ab87ae06..4af9e31ee 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -474,6 +474,6 @@ def test_create_push_subscription_with_subscription_suffix(self): self.assertIsNone(result.exception) self.assertEqual(result.exit_code, 0) - self.assertEqual(subscription.call_args.kwargs["topic"].name, "octue.services.octue.example-service.3-5-0") + self.assertEqual(subscription.call_args.kwargs["topic"].name, "octue.example-service.3-5-0") self.assertEqual(subscription.call_args.kwargs["name"], "octue.example-service.3-5-0-peter-rabbit") self.assertEqual(subscription.call_args.kwargs["push_endpoint"], "https://example.com/endpoint") From da45d91175955f168ffd595050cc029ee01ab0c5 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 2 Apr 2024 18:58:35 +0100 Subject: [PATCH 099/169] FIX: Shorten answer subscription filter skipci --- octue/cloud/pub_sub/service.py | 1 - 1 file changed, 1 deletion(-) diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index 6dea282fb..35e12356a 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -339,7 +339,6 @@ def ask( name=".".join((self.backend.services_namespace, ANSWERS_NAMESPACE, pub_sub_id, question_uuid)), topic=services_topic, filter=( - f'attributes.sender = "{service_id}" ' f'attributes.recipient = "{self.id}" ' f'AND attributes.question_uuid = "{question_uuid}" ' f'AND attributes.sender_type = "{CHILD_SENDER_TYPE}"' From 1140979664c8985e7d066aaf22e8b98b779f9c6d Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 8 Apr 2024 10:18:06 +0100 Subject: [PATCH 100/169] TST: Update child emulator tests skipci --- tests/cloud/emulators/test_child_emulator.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/cloud/emulators/test_child_emulator.py b/tests/cloud/emulators/test_child_emulator.py index 893d7731f..d2c6a4fa3 100644 --- a/tests/cloud/emulators/test_child_emulator.py +++ b/tests/cloud/emulators/test_child_emulator.py @@ -2,12 +2,12 @@ import os from octue.cloud import storage -from octue.cloud.emulators._pub_sub import MockService +from octue.cloud.emulators._pub_sub import MockService, MockTopic from octue.cloud.emulators.child import ChildEmulator, ServicePatcher from octue.cloud.storage import GoogleCloudStorageClient from octue.resources import Manifest from octue.resources.service_backends import GCPPubSubBackend -from tests import MOCK_SERVICE_REVISION_TAG, TEST_BUCKET_NAME, TESTS_DIR +from tests import MOCK_SERVICE_REVISION_TAG, TEST_BUCKET_NAME, TEST_PROJECT_NAME, TESTS_DIR from tests.base import BaseTestCase @@ -15,6 +15,13 @@ class TestChildEmulatorAsk(BaseTestCase): BACKEND = {"name": "GCPPubSubBackend", "project_name": "blah"} + @classmethod + def setUpClass(cls): + topic = MockTopic(name="octue.services", project_name=TEST_PROJECT_NAME) + + with ServicePatcher(): + topic.create(allow_existing=True) + def test_representation(self): """Test that child emulators are represented correctly.""" self.assertEqual( @@ -297,6 +304,13 @@ class TestChildEmulatorJSONFiles(BaseTestCase): TEST_FILES_DIRECTORY = os.path.join(TESTS_DIR, "cloud", "emulators", "valid_child_emulator_files") + @classmethod + def setUpClass(cls): + topic = MockTopic(name="octue.services", project_name=TEST_PROJECT_NAME) + + with ServicePatcher(): + topic.create(allow_existing=True) + def test_with_empty_file(self): """Test that a child emulator can be instantiated from an empty JSON file (a JSON file with only an empty object in), asked a question, and produce a trivial result. From 6ea4668c82d8b239f089d163f88a49e0ba6c3849 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 8 Apr 2024 10:45:28 +0100 Subject: [PATCH 101/169] ENH: Update CLI command to use single topic per workspace skipci --- octue/cli.py | 28 +++++---------- octue/cloud/pub_sub/__init__.py | 19 +++------- tests/test_cli.py | 61 ++++----------------------------- 3 files changed, 19 insertions(+), 89 deletions(-) diff --git a/octue/cli.py b/octue/cli.py index 9d649b597..24fad367d 100644 --- a/octue/cli.py +++ b/octue/cli.py @@ -10,6 +10,7 @@ import click from google import auth +from octue import DEFAULT_OCTUE_SERVICES_NAMESPACE from octue.cloud import pub_sub, storage from octue.cloud.pub_sub.service import Service from octue.cloud.service_id import create_sruid, get_sruid_parts @@ -358,21 +359,11 @@ def deploy(): ". 'curious-capybara'.", ) @click.option( - "--filter", + "--services-namespace", is_flag=False, - default='attributes.sender_type = "PARENT"', + default=DEFAULT_OCTUE_SERVICES_NAMESPACE, show_default=True, - help="An optional filter to apply to the subscription (see " - "https://cloud.google.com/pubsub/docs/subscription-message-filter). If not provided, the default filter is applied." - " To disable filtering, provide an empty string.", -) -@click.option( - "--subscription-suffix", - is_flag=False, - default=None, - show_default=True, - help="An optional suffix to add to the end of the subscription name. This is useful when needing to create " - "multiple subscriptions for the same topic (subscription names are unique).", + help="The services namespace to emit and consume events from.", ) def create_push_subscription( project_name, @@ -381,11 +372,10 @@ def create_push_subscription( push_endpoint, expiration_time, revision_tag, - filter, - subscription_suffix, + services_namespace, ): - """Create a Google Pub/Sub push subscription for an Octue service for it to receive questions from parents. If a - corresponding topic doesn't exist, it will be created first. The subscription name is printed on completion. + """Create a Google Pub/Sub push subscription for an Octue service for it to receive questions from parents. The + subscription name is printed on completion. PROJECT_NAME is the name of the Google Cloud project in which the subscription will be created @@ -403,8 +393,8 @@ def create_push_subscription( sruid, push_endpoint, expiration_time=expiration_time, - subscription_filter=filter or None, - subscription_suffix=subscription_suffix, + subscription_filter=f'attributes.recipient = "{sruid}" AND attributes.sender_type = "PARENT"', + services_namespace=services_namespace, ) click.echo(sruid) diff --git a/octue/cloud/pub_sub/__init__.py b/octue/cloud/pub_sub/__init__.py index ab0cccf99..b8e87bc01 100644 --- a/octue/cloud/pub_sub/__init__.py +++ b/octue/cloud/pub_sub/__init__.py @@ -13,7 +13,7 @@ def create_push_subscription( push_endpoint, subscription_filter=None, expiration_time=None, - subscription_suffix=None, + services_namespace="octue.services", ): """Create a Google Pub/Sub push subscription for an Octue service for it to receive questions from parents. If a corresponding topic doesn't exist, it will be created first. @@ -23,28 +23,17 @@ def create_push_subscription( :param str push_endpoint: the HTTP/HTTPS endpoint of the service to push to. It should be fully formed and include the 'https://' prefix :param str|None subscription_filter: if specified, the filter to apply to the subscription; otherwise, no filter is applied :param float|None expiration_time: the number of seconds of inactivity after which the subscription should expire. If not provided, no expiration time is applied to the subscription - :param str|None subscription_suffix: if provided, add a suffix to the end of the subscription name. This is useful when needing to create multiple subscriptions for the same topic (subscription names are unique). + :param str services_namespace: the services namespace to emit and consume events from :return None: """ - topic_name = convert_service_id_to_pub_sub_form(sruid) - - if subscription_suffix: - subscription_name = topic_name + subscription_suffix - else: - subscription_name = topic_name - - topic = Topic(name=topic_name, project_name=project_name) - topic.create(allow_existing=True) - if expiration_time: expiration_time = float(expiration_time) else: expiration_time = None subscription = Subscription( - name=subscription_name, - topic=topic, - project_name=project_name, + name=convert_service_id_to_pub_sub_form(sruid), + topic=Topic(name=services_namespace, project_name=project_name), filter=subscription_filter, expiration_time=expiration_time, push_endpoint=push_endpoint, diff --git a/tests/test_cli.py b/tests/test_cli.py index 4af9e31ee..2b8f59028 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -14,7 +14,7 @@ from octue.configuration import AppConfiguration, ServiceConfiguration from octue.resources import Dataset from octue.utils.patches import MultiPatcher -from tests import MOCK_SERVICE_REVISION_TAG, TEST_BUCKET_NAME, TESTS_DIR +from tests import MOCK_SERVICE_REVISION_TAG, TEST_BUCKET_NAME, TEST_PROJECT_NAME, TESTS_DIR from tests.base import BaseTestCase @@ -194,6 +194,11 @@ def setUpClass(cls): }, ) + topic = MockTopic(name="octue.services", project_name=TEST_PROJECT_NAME) + + with ServicePatcher(): + topic.create(allow_existing=True) + def test_start_command(self): """Test that the start command works without error and uses the revision tag supplied in the `OCTUE_SERVICE_REVISION_TAG` environment variable. @@ -423,57 +428,3 @@ def test_create_push_subscription(self): self.assertEqual(subscription.call_args.kwargs["name"], "octue.example-service.3-5-0") self.assertEqual(subscription.call_args.kwargs["push_endpoint"], "https://example.com/endpoint") self.assertEqual(subscription.call_args.kwargs["expiration_time"], expected_expiration_time) - - def test_create_push_subscription_with_filter(self): - """Test that filters are added to subscriptions correctly when creating a push subscription.""" - for filter_option, expected_filter in ( - ([], 'attributes.sender_type = "PARENT"'), - (["--filter="], None), - (['--filter=attributes.sender_type = "CHILD"'], 'attributes.sender_type = "CHILD"'), - ): - with self.subTest(filter_option=filter_option): - with patch("octue.cloud.pub_sub.Topic", new=MockTopic): - with patch("octue.cloud.pub_sub.Subscription") as subscription: - result = CliRunner().invoke( - octue_cli, - [ - "deploy", - "create-push-subscription", - "my-project", - "octue", - "example-service", - "https://example.com/endpoint", - "--revision-tag=3.5.0", - *filter_option, - ], - ) - - self.assertIsNone(result.exception) - self.assertEqual(result.exit_code, 0) - self.assertEqual(subscription.call_args.kwargs["name"], "octue.example-service.3-5-0") - self.assertEqual(subscription.call_args.kwargs["push_endpoint"], "https://example.com/endpoint") - self.assertEqual(subscription.call_args.kwargs["filter"], expected_filter) - - def test_create_push_subscription_with_subscription_suffix(self): - """Test that subscription suffixes are added to subscription names correctly when creating a push subscription.""" - with patch("octue.cloud.pub_sub.Topic", new=MockTopic): - with patch("octue.cloud.pub_sub.Subscription") as subscription: - result = CliRunner().invoke( - octue_cli, - [ - "deploy", - "create-push-subscription", - "my-project", - "octue", - "example-service", - "https://example.com/endpoint", - "--revision-tag=3.5.0", - "--subscription-suffix=-peter-rabbit", - ], - ) - - self.assertIsNone(result.exception) - self.assertEqual(result.exit_code, 0) - self.assertEqual(subscription.call_args.kwargs["topic"].name, "octue.example-service.3-5-0") - self.assertEqual(subscription.call_args.kwargs["name"], "octue.example-service.3-5-0-peter-rabbit") - self.assertEqual(subscription.call_args.kwargs["push_endpoint"], "https://example.com/endpoint") From e61013a81a4011390170f4920ae8b2d69c90e293 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 8 Apr 2024 11:23:23 +0100 Subject: [PATCH 102/169] DOC: Update docstrings --- octue/cli.py | 2 +- octue/cloud/pub_sub/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/octue/cli.py b/octue/cli.py index 24fad367d..985c25f5b 100644 --- a/octue/cli.py +++ b/octue/cli.py @@ -363,7 +363,7 @@ def deploy(): is_flag=False, default=DEFAULT_OCTUE_SERVICES_NAMESPACE, show_default=True, - help="The services namespace to emit and consume events from.", + help="The services namespace to subscribe to.", ) def create_push_subscription( project_name, diff --git a/octue/cloud/pub_sub/__init__.py b/octue/cloud/pub_sub/__init__.py index b8e87bc01..1b9f63f5e 100644 --- a/octue/cloud/pub_sub/__init__.py +++ b/octue/cloud/pub_sub/__init__.py @@ -23,7 +23,7 @@ def create_push_subscription( :param str push_endpoint: the HTTP/HTTPS endpoint of the service to push to. It should be fully formed and include the 'https://' prefix :param str|None subscription_filter: if specified, the filter to apply to the subscription; otherwise, no filter is applied :param float|None expiration_time: the number of seconds of inactivity after which the subscription should expire. If not provided, no expiration time is applied to the subscription - :param str services_namespace: the services namespace to emit and consume events from + :param str services_namespace: the services namespace to subscribe to :return None: """ if expiration_time: From 7472cd9c72c3f60c85b5edcad8d3ecd3ef086bff Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 8 Apr 2024 11:23:40 +0100 Subject: [PATCH 103/169] ENH: Use latest services communication schema --- octue/cloud/events/validation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octue/cloud/events/validation.py b/octue/cloud/events/validation.py index 553018401..d8cd1d802 100644 --- a/octue/cloud/events/validation.py +++ b/octue/cloud/events/validation.py @@ -8,7 +8,7 @@ logger = logging.getLogger(__name__) -SERVICE_COMMUNICATION_SCHEMA = {"$ref": "https://jsonschema.registry.octue.com/octue/service-communication/0.8.3.json"} +SERVICE_COMMUNICATION_SCHEMA = {"$ref": "https://jsonschema.registry.octue.com/octue/service-communication/0.9.0.json"} SERVICE_COMMUNICATION_SCHEMA_INFO_URL = "https://strands.octue.com/octue/service-communication" SERVICE_COMMUNICATION_SCHEMA_VERSION = os.path.splitext(SERVICE_COMMUNICATION_SCHEMA["$ref"])[0].split("/")[-1] From 57cff9e14901c19fe85d19f2308b4287f9a67ab0 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 8 Apr 2024 11:42:16 +0100 Subject: [PATCH 104/169] FIX: Update `get_sruid_from_pub_sub_resource_name` --- octue/cloud/service_id.py | 2 +- tests/cloud/test_service_id.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/octue/cloud/service_id.py b/octue/cloud/service_id.py index 60852e82b..dd041a38a 100644 --- a/octue/cloud/service_id.py +++ b/octue/cloud/service_id.py @@ -213,7 +213,7 @@ def get_sruid_from_pub_sub_resource_name(name): :param str name: the name of the topic or subscription :return str: the SRUID of the service revision the topic or subscription is related to """ - _, _, namespace, name, revision_tag, *_ = name.split(".") + namespace, name, revision_tag, *_ = name.split(".") return f"{namespace}/{name}:{revision_tag.replace('-', '.')}" diff --git a/tests/cloud/test_service_id.py b/tests/cloud/test_service_id.py index 5e637227d..606e76dfa 100644 --- a/tests/cloud/test_service_id.py +++ b/tests/cloud/test_service_id.py @@ -84,7 +84,6 @@ def test_convert_service_id_to_pub_sub_form(self): ("octue/my-service", "octue.my-service"), ("octue/my-service:0.1.7", "octue.my-service.0-1-7"), ("my-service:3.1.9", "my-service.3-1-9"), - ("octue.services.octue/my-service:0.1.7", "octue.services.octue.my-service.0-1-7"), ) for service_id, pub_sub_service_id in service_ids: @@ -95,7 +94,7 @@ def test_convert_service_id_to_pub_sub_form(self): class TestGetSRUIDFromPubSubResourceName(unittest.TestCase): def test_get_sruid_from_pub_sub_resource_name(self): """Test that an SRUID can be extracted from a Pub/Sub resource name.""" - sruid = get_sruid_from_pub_sub_resource_name("octue.services.octue.example-service-cloud-run.0-3-2") + sruid = get_sruid_from_pub_sub_resource_name("octue.example-service-cloud-run.0-3-2") self.assertEqual(sruid, "octue/example-service-cloud-run:0.3.2") From feadb813e2550de410e36fb1226209b1c5227595 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 8 Apr 2024 11:42:31 +0100 Subject: [PATCH 105/169] TST: Remove unnecessary `octue.services` --- tests/cloud/pub_sub/test_events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cloud/pub_sub/test_events.py b/tests/cloud/pub_sub/test_events.py index 386f6dd56..7edb62008 100644 --- a/tests/cloud/pub_sub/test_events.py +++ b/tests/cloud/pub_sub/test_events.py @@ -21,7 +21,7 @@ def create_mock_topic_and_subscription(): topic = MockTopic(name="octue.services", project_name=TEST_PROJECT_NAME) topic.create(allow_existing=True) - subscription = MockSubscription(name=f"octue.services.my-org.my-service.1-0-0.answers.{question_uuid}", topic=topic) + subscription = MockSubscription(name=f"my-org.my-service.1-0-0.answers.{question_uuid}", topic=topic) subscription.create() with ServicePatcher(): From a25d9194aee1af599332bd7e6b251fdc316c7e00 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 8 Apr 2024 11:50:58 +0100 Subject: [PATCH 106/169] REF: Move event counter out of `Topic` and into `Service` --- octue/cloud/pub_sub/service.py | 20 ++++++++++++-------- octue/cloud/pub_sub/topic.py | 1 - tests/cloud/pub_sub/test_events.py | 14 +++++++------- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index 35e12356a..53f0eca22 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -35,10 +35,10 @@ logger = logging.getLogger(__name__) -# A lock to ensure only one message can be sent at a time so that the order is incremented correctly when messages are -# being sent on multiple threads (e.g. via the main thread and a periodic monitor message thread). This avoids 1) -# messages overwriting each other in the parent's message handler and 2) messages losing their order. -send_message_lock = threading.Lock() +# A lock to ensure only one event can be emitted at a time so that the order is incremented correctly when events are +# being emitted on multiple threads (e.g. via the main thread and a periodic monitor message thread). This avoids 1) +# events overwriting each other in the parent's message handler and 2) events losing their order. +emit_event_lock = threading.Lock() DEFAULT_NAMESPACE = "default" ANSWERS_NAMESPACE = "answers" @@ -92,6 +92,7 @@ def __init__(self, backend, service_id=None, run_function=None, name=None, servi self._local_sdk_version = importlib.metadata.version("octue") self._publisher = None self._event_handler = None + self._events_emitted = 0 def __repr__(self): """Represent the service as a string. @@ -207,6 +208,7 @@ def answer(self, question, heartbeat_interval=120, timeout=30): heartbeater = None services_topic = self._get_services_topic() + self._events_emitted = 0 try: self._send_delivery_acknowledgment(services_topic, question_uuid, originator) @@ -347,6 +349,8 @@ def ask( ) answer_subscription.create(allow_existing=False) + self._events_emitted = 0 + self._send_question( input_values=input_values, input_manifest=input_manifest, @@ -438,7 +442,7 @@ def _get_services_topic(self): def _send_message(self, message, topic, originator, recipient, attributes=None, timeout=30): """Send a JSON-serialised message to the given topic with optional message attributes and increment the - `messages_published` attribute of the topic by one. This method is thread-safe. + `_events_emitted` attribute by one. This method is thread-safe. :param dict message: JSON-serialisable data to send as a message :param octue.cloud.pub_sub.topic.Topic topic: the Pub/Sub topic to send the message to @@ -453,9 +457,9 @@ def _send_message(self, message, topic, originator, recipient, attributes=None, attributes["sender"] = self.id attributes["recipient"] = recipient - with send_message_lock: + with emit_event_lock: attributes["sender_sdk_version"] = self._local_sdk_version - attributes["order"] = topic.messages_published + attributes["order"] = self._events_emitted attributes["uuid"] = str(uuid.uuid4()) converted_attributes = {} @@ -475,7 +479,7 @@ def _send_message(self, message, topic, originator, recipient, attributes=None, **converted_attributes, ) - topic.messages_published += 1 + self._events_emitted += 1 return future diff --git a/octue/cloud/pub_sub/topic.py b/octue/cloud/pub_sub/topic.py index 0af113c99..18c2947a4 100644 --- a/octue/cloud/pub_sub/topic.py +++ b/octue/cloud/pub_sub/topic.py @@ -24,7 +24,6 @@ def __init__(self, name, project_name): self.name = name self.project_name = project_name self.path = self.generate_topic_path(self.project_name, self.name) - self.messages_published = 0 self._publisher = PublisherClient() self._created = False diff --git a/tests/cloud/pub_sub/test_events.py b/tests/cloud/pub_sub/test_events.py index 7edb62008..92b7d43e8 100644 --- a/tests/cloud/pub_sub/test_events.py +++ b/tests/cloud/pub_sub/test_events.py @@ -120,7 +120,7 @@ def test_out_of_order_messages_are_handled_in_order(self): ] for message in messages: - mock_topic.messages_published = message["event"]["order"] + child._events_emitted = message["event"]["order"] child._send_message( message=message["event"], attributes=message["attributes"], @@ -179,7 +179,7 @@ def test_out_of_order_messages_with_end_message_first_are_handled_in_order(self) ] for message in messages: - mock_topic.messages_published = message["event"]["order"] + child._events_emitted = message["event"]["order"] child._send_message( message=message["event"], attributes=message["attributes"], @@ -386,7 +386,7 @@ def test_missing_messages_at_start_can_be_skipped(self): child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) # Simulate the first two messages not being received. - mock_topic.messages_published = 2 + child._events_emitted = 2 messages = [ { @@ -470,7 +470,7 @@ def test_missing_messages_in_middle_can_skipped(self): ) # Simulate missing messages. - mock_topic.messages_published = 5 + child._events_emitted = 5 # Send a final message. child._send_message( @@ -535,7 +535,7 @@ def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): ) # Simulate missing messages. - mock_topic.messages_published = 5 + child._events_emitted = 5 # Send another message. child._send_message( @@ -547,7 +547,7 @@ def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): ) # Simulate more missing messages. - mock_topic.messages_published = 20 + child._events_emitted = 20 # Send more consecutive messages. messages = [ @@ -611,7 +611,7 @@ def test_all_messages_missing_apart_from_result(self): child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) # Simulate missing messages. - mock_topic.messages_published = 1000 + child._events_emitted = 1000 # Send the result message. child._send_message( From 09fc8095ee8dc67493cc72adf49bf6f6c0cc172b Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 8 Apr 2024 11:53:21 +0100 Subject: [PATCH 107/169] REF: Move unrelated code out of thread lock --- octue/cloud/pub_sub/service.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index 53f0eca22..037496b2d 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -453,15 +453,14 @@ def _send_message(self, message, topic, originator, recipient, attributes=None, :return google.cloud.pubsub_v1.publisher.futures.Future: """ attributes = attributes or {} + attributes["uuid"] = str(uuid.uuid4()) attributes["originator"] = originator attributes["sender"] = self.id + attributes["sender_sdk_version"] = self._local_sdk_version attributes["recipient"] = recipient with emit_event_lock: - attributes["sender_sdk_version"] = self._local_sdk_version attributes["order"] = self._events_emitted - attributes["uuid"] = str(uuid.uuid4()) - converted_attributes = {} for key, value in attributes.items(): From e401a71810c7d620f9051bf0f754e210c6fa969e Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 8 Apr 2024 12:03:18 +0100 Subject: [PATCH 108/169] REF: Factor out services topic in `Service` --- .../google/answer_pub_sub_question.py | 9 +--- octue/cloud/pub_sub/logging.py | 5 +-- octue/cloud/pub_sub/service.py | 42 ++++++------------- 3 files changed, 15 insertions(+), 41 deletions(-) diff --git a/octue/cloud/deployment/google/answer_pub_sub_question.py b/octue/cloud/deployment/google/answer_pub_sub_question.py index 167832582..703a67f5c 100644 --- a/octue/cloud/deployment/google/answer_pub_sub_question.py +++ b/octue/cloud/deployment/google/answer_pub_sub_question.py @@ -1,8 +1,7 @@ import logging -from octue.cloud.pub_sub import Topic from octue.cloud.pub_sub.service import Service -from octue.cloud.service_id import convert_service_id_to_pub_sub_form, create_sruid, get_sruid_parts +from octue.cloud.service_id import create_sruid, get_sruid_parts from octue.configuration import load_service_and_app_configuration from octue.resources.service_backends import GCPPubSubBackend from octue.runner import Runner @@ -47,9 +46,5 @@ def answer_question(question, project_name): logger.info("Analysis successfully run and response sent for question %r.", question_uuid) except BaseException as error: # noqa - service.send_exception( - topic=Topic(name=convert_service_id_to_pub_sub_form(service_sruid), project_name=project_name), - question_uuid=question_uuid, - ) - + service.send_exception(question_uuid=question_uuid, originator="UNKNOWN") logger.exception(error) diff --git a/octue/cloud/pub_sub/logging.py b/octue/cloud/pub_sub/logging.py index 4e091dc84..f28906702 100644 --- a/octue/cloud/pub_sub/logging.py +++ b/octue/cloud/pub_sub/logging.py @@ -9,7 +9,6 @@ class GoogleCloudPubSubHandler(logging.Handler): """A log handler that publishes log records to a Google Cloud Pub/Sub topic. :param callable message_sender: the `_send_message` method of the service that instantiated this instance - :param octue.cloud.pub_sub.topic.Topic topic: topic to publish log records to :param str question_uuid: the UUID of the question to handle log records for :param str originator: the SRUID of the service that asked the question these log records are related to :param str recipient: the SRUID of the service to send these log records to @@ -17,9 +16,8 @@ class GoogleCloudPubSubHandler(logging.Handler): :return None: """ - def __init__(self, message_sender, topic, question_uuid, originator, recipient, timeout=60, *args, **kwargs): + def __init__(self, message_sender, question_uuid, originator, recipient, timeout=60, *args, **kwargs): super().__init__(*args, **kwargs) - self.topic = topic self.question_uuid = question_uuid self.originator = originator self.recipient = recipient @@ -38,7 +36,6 @@ def emit(self, record): "kind": "log_record", "log_record": self._convert_log_record_to_primitives(record), }, - topic=self.topic, originator=self.originator, recipient=self.recipient, attributes={ diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index 037496b2d..48c0b7550 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -86,6 +86,7 @@ def __init__(self, backend, service_id=None, run_function=None, name=None, servi self.run_function = run_function self.name = name + self.services_topic = self._get_services_topic() self.service_registries = service_registries self._pub_sub_id = convert_service_id_to_pub_sub_form(self.id) @@ -140,7 +141,7 @@ def serve(self, timeout=None, delete_topic_and_subscription_on_exit=False, allow subscription = Subscription( name=".".join((self.backend.services_namespace, self._pub_sub_id)), - topic=self._get_services_topic(), + topic=self.services_topic, filter=f'attributes.recipient = "{self.id}" AND attributes.sender_type = "{PARENT_SENDER_TYPE}"', expiration_time=None, ) @@ -206,17 +207,15 @@ def answer(self, question, heartbeat_interval=120, timeout=30): return heartbeater = None - - services_topic = self._get_services_topic() self._events_emitted = 0 try: - self._send_delivery_acknowledgment(services_topic, question_uuid, originator) + self._send_delivery_acknowledgment(question_uuid, originator) heartbeater = RepeatingTimer( interval=heartbeat_interval, function=self._send_heartbeat, - kwargs={"topic": services_topic, "question_uuid": question_uuid, "originator": originator}, + kwargs={"question_uuid": question_uuid, "originator": originator}, ) heartbeater.daemon = True @@ -225,7 +224,6 @@ def answer(self, question, heartbeat_interval=120, timeout=30): if forward_logs: analysis_log_handler = GoogleCloudPubSubHandler( message_sender=self._send_message, - topic=services_topic, question_uuid=question_uuid, originator=originator, recipient=originator, @@ -241,7 +239,6 @@ def answer(self, question, heartbeat_interval=120, timeout=30): analysis_log_handler=analysis_log_handler, handle_monitor_message=functools.partial( self._send_monitor_message, - topic=services_topic, question_uuid=question_uuid, originator=originator, ), @@ -255,7 +252,6 @@ def answer(self, question, heartbeat_interval=120, timeout=30): self._send_message( message=result, - topic=services_topic, originator=originator, recipient=originator, attributes={"question_uuid": question_uuid, "sender_type": CHILD_SENDER_TYPE}, @@ -270,7 +266,7 @@ def answer(self, question, heartbeat_interval=120, timeout=30): heartbeater.cancel() warn_if_incompatible(child_sdk_version=self._local_sdk_version, parent_sdk_version=parent_sdk_version) - self.send_exception(services_topic, question_uuid, originator, timeout=timeout) + self.send_exception(question_uuid, originator, timeout=timeout) raise error def ask( @@ -330,7 +326,6 @@ def ask( ) question_uuid = question_uuid or str(uuid.uuid4()) - services_topic = self._get_services_topic() if asynchronous and not push_endpoint: answer_subscription = None @@ -339,7 +334,7 @@ def ask( answer_subscription = Subscription( name=".".join((self.backend.services_namespace, ANSWERS_NAMESPACE, pub_sub_id, question_uuid)), - topic=services_topic, + topic=self.services_topic, filter=( f'attributes.recipient = "{self.id}" ' f'AND attributes.question_uuid = "{question_uuid}" ' @@ -358,7 +353,6 @@ def ask( service_id=service_id, forward_logs=subscribe_to_logs, save_diagnostics=save_diagnostics, - topic=services_topic, question_uuid=question_uuid, ) @@ -406,10 +400,9 @@ def wait_for_answer( finally: subscription.delete() - def send_exception(self, topic, question_uuid, originator, timeout=30): + def send_exception(self, question_uuid, originator, timeout=30): """Serialise and send the exception being handled to the parent. - :param octue.cloud.pub_sub.topic.Topic topic: :param str question_uuid: :param str originator: the SRUID of the service that asked the question this event is related to :param float|None timeout: time in seconds to keep retrying sending of the exception @@ -425,7 +418,6 @@ def send_exception(self, topic, question_uuid, originator, timeout=30): "exception_message": exception_message, "exception_traceback": exception["traceback"], }, - topic=topic, originator=originator, recipient=originator, attributes={"question_uuid": question_uuid, "sender_type": CHILD_SENDER_TYPE}, @@ -440,12 +432,11 @@ def _get_services_topic(self): return topic - def _send_message(self, message, topic, originator, recipient, attributes=None, timeout=30): + def _send_message(self, message, originator, recipient, attributes=None, timeout=30): """Send a JSON-serialised message to the given topic with optional message attributes and increment the `_events_emitted` attribute by one. This method is thread-safe. :param dict message: JSON-serialisable data to send as a message - :param octue.cloud.pub_sub.topic.Topic topic: the Pub/Sub topic to send the message to :param str originator: the SRUID of the service that asked the question this event is related to :param str recipient: :param dict|None attributes: key-value pairs to attach to the message - the values must be strings or bytes @@ -472,7 +463,7 @@ def _send_message(self, message, topic, originator, recipient, attributes=None, converted_attributes[key] = value future = self.publisher.publish( - topic=topic.path, + topic=self.services_topic.path, data=json.dumps(message, cls=OctueJSONEncoder).encode(), retry=retry.Retry(deadline=timeout), **converted_attributes, @@ -490,7 +481,6 @@ def _send_question( service_id, forward_logs, save_diagnostics, - topic, question_uuid, timeout=30, ): @@ -502,7 +492,6 @@ def _send_question( :param str service_id: the ID of the child to send the question to :param bool forward_logs: whether to request the child to forward its logs :param str save_diagnostics: must be one of {"SAVE_DIAGNOSTICS_OFF", "SAVE_DIAGNOSTICS_ON_CRASH", "SAVE_DIAGNOSTICS_ON"}; if turned on, allow the input values and manifest (and its datasets) to be saved by the child either all the time or just if it fails while processing them - :param topic: :param str question_uuid: the UUID of the question being sent :param float timeout: time in seconds after which to give up sending :return None: @@ -515,7 +504,6 @@ def _send_question( future = self._send_message( message=question, - topic=topic, timeout=timeout, originator=self.id, recipient=service_id, @@ -531,10 +519,9 @@ def _send_question( future.result() logger.info("%r asked a question %r to service %r.", self, question_uuid, service_id) - def _send_delivery_acknowledgment(self, topic, question_uuid, originator, timeout=30): + def _send_delivery_acknowledgment(self, question_uuid, originator, timeout=30): """Send an acknowledgement of question receipt to the parent. - :param octue.cloud.pub_sub.topic.Topic topic: topic to send the acknowledgement to :param str question_uuid: :param str originator: the SRUID of the service that asked the question this event is related to :param float timeout: time in seconds after which to give up sending @@ -545,7 +532,6 @@ def _send_delivery_acknowledgment(self, topic, question_uuid, originator, timeou "kind": "delivery_acknowledgement", "datetime": datetime.datetime.utcnow().isoformat(), }, - topic=topic, timeout=timeout, originator=originator, recipient=originator, @@ -554,10 +540,9 @@ def _send_delivery_acknowledgment(self, topic, question_uuid, originator, timeou logger.info("%r acknowledged receipt of question %r.", self, question_uuid) - def _send_heartbeat(self, topic, question_uuid, originator, timeout=30): + def _send_heartbeat(self, question_uuid, originator, timeout=30): """Send a heartbeat to the parent, indicating that the service is alive. - :param octue.cloud.pub_sub.topic.Topic topic: topic to send the heartbeat to :param str question_uuid: :param str originator: the SRUID of the service that asked the question this event is related to :param float timeout: time in seconds after which to give up sending @@ -568,7 +553,6 @@ def _send_heartbeat(self, topic, question_uuid, originator, timeout=30): "kind": "heartbeat", "datetime": datetime.datetime.utcnow().isoformat(), }, - topic=topic, originator=originator, recipient=originator, timeout=timeout, @@ -577,11 +561,10 @@ def _send_heartbeat(self, topic, question_uuid, originator, timeout=30): logger.debug("Heartbeat sent by %r.", self) - def _send_monitor_message(self, data, topic, question_uuid, originator, timeout=30): + def _send_monitor_message(self, data, question_uuid, originator, timeout=30): """Send a monitor message to the parent. :param any data: the data to send as a monitor message - :param octue.cloud.pub_sub.topic.Topic topic: the topic to send the message to :param str question_uuid: :param str originator: the SRUID of the service that asked the question this event is related to :param float timeout: time in seconds to retry sending the message @@ -589,7 +572,6 @@ def _send_monitor_message(self, data, topic, question_uuid, originator, timeout= """ self._send_message( {"kind": "monitor_message", "data": data}, - topic=topic, originator=originator, recipient=originator, timeout=timeout, From 1e5d6348e5b5b47c24348f128ea349886d7a8498 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 8 Apr 2024 16:01:58 +0100 Subject: [PATCH 109/169] OPS: Update bigquery table and event handler cloud function --- terraform/bigquery.tf | 21 ++++++++++++++++++--- terraform/functions.tf | 19 +++++++------------ 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/terraform/bigquery.tf b/terraform/bigquery.tf index d48391221..a4cb12b98 100644 --- a/terraform/bigquery.tf +++ b/terraform/bigquery.tf @@ -28,6 +28,16 @@ resource "google_bigquery_table" "test_table" { "type": "JSON", "mode": "REQUIRED" }, + { + "name": "uuid", + "type": "STRING", + "mode": "REQUIRED" + }, + { + "name": "originator", + "type": "STRING", + "mode": "REQUIRED" + }, { "name": "sender", "type": "STRING", @@ -39,17 +49,22 @@ resource "google_bigquery_table" "test_table" { "mode": "REQUIRED" }, { - "name": "question_uuid", + "name": "sender_sdk_version", "type": "STRING", "mode": "REQUIRED" }, { - "name": "version", + "name": "recipient", + "type": "STRING", + "mode": "REQUIRED" + }, + { + "name": "question_uuid", "type": "STRING", "mode": "REQUIRED" }, { - "name": "ordering_key", + "name": "order", "type": "STRING", "mode": "REQUIRED" }, diff --git a/terraform/functions.tf b/terraform/functions.tf index cd9d7b13f..3f90238c1 100644 --- a/terraform/functions.tf +++ b/terraform/functions.tf @@ -9,7 +9,7 @@ resource "google_cloudfunctions2_function" "event_handler" { source { storage_source { bucket = "twined-gcp" - object = "event_handler_function_source.zip" + object = "event_handler/0.2.0.zip" } } } @@ -23,12 +23,12 @@ resource "google_cloudfunctions2_function" "event_handler" { } } -# event_trigger { -# trigger_region = var.region -# event_type = "google.cloud.pubsub.topic.v1.messagePublished" -# pubsub_topic = google_pubsub_topic.topic.id -# retry_policy = "RETRY_POLICY_RETRY" -# } + event_trigger { + trigger_region = var.region + event_type = "google.cloud.pubsub.topic.v1.messagePublished" + pubsub_topic = google_pubsub_topic.services_topic.id + retry_policy = "RETRY_POLICY_RETRY" + } } @@ -39,8 +39,3 @@ resource "google_cloud_run_service_iam_member" "function_invoker" { role = "roles/run.invoker" member = "allUsers" } - - -output "function_uri" { - value = google_cloudfunctions2_function.event_handler.service_config[0].uri -} From 5b89aafc62d5178c3a30c5a4bdb00a33859f1cd3 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 9 Apr 2024 11:36:12 +0100 Subject: [PATCH 110/169] FIX: Restore answer subscription names to previous format --- octue/cloud/pub_sub/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index 48c0b7550..4c7d8c185 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -333,7 +333,7 @@ def ask( pub_sub_id = convert_service_id_to_pub_sub_form(self.id) answer_subscription = Subscription( - name=".".join((self.backend.services_namespace, ANSWERS_NAMESPACE, pub_sub_id, question_uuid)), + name=".".join((self.backend.services_namespace, pub_sub_id, ANSWERS_NAMESPACE, question_uuid)), topic=self.services_topic, filter=( f'attributes.recipient = "{self.id}" ' From ddf7100bda6f6c91065800ea0132fa954237c592 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 9 Apr 2024 11:48:58 +0100 Subject: [PATCH 111/169] DOC: Update `Service` docstrings --- octue/cloud/pub_sub/service.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index 4c7d8c185..346da3148 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -425,6 +425,11 @@ def send_exception(self, question_uuid, originator, timeout=30): ) def _get_services_topic(self): + """Get the Octue services topic that all events in the project are published to. + + :raise octue.exceptions.ServiceNotFound: if the topic doesn't exist in the project + :return octue.cloud.pub_sub.topic.Topic: the Octue services topic for the project + """ topic = Topic(name=self.backend.services_namespace, project_name=self.backend.project_name) if not topic.exists(): @@ -433,14 +438,14 @@ def _get_services_topic(self): return topic def _send_message(self, message, originator, recipient, attributes=None, timeout=30): - """Send a JSON-serialised message to the given topic with optional message attributes and increment the - `_events_emitted` attribute by one. This method is thread-safe. + """Send a JSON-serialised event as a Pub/Sub message to the services topic with optional message attributes, + incrementing the `_events_emitted` attribute by one. This method is thread-safe. :param dict message: JSON-serialisable data to send as a message :param str originator: the SRUID of the service that asked the question this event is related to - :param str recipient: - :param dict|None attributes: key-value pairs to attach to the message - the values must be strings or bytes - :param int|float timeout: the timeout for sending the message in seconds + :param str recipient: the SRUID of the service the event is intended for + :param dict|None attributes: key-value pairs to attach to the event - the values must be strings or bytes + :param int|float timeout: the timeout for sending the event in seconds :return google.cloud.pubsub_v1.publisher.futures.Future: """ attributes = attributes or {} From 82db91e08ca86734bf59b622568ba2a01a9fc20d Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 9 Apr 2024 11:49:41 +0100 Subject: [PATCH 112/169] TST: Simplify `GoogleCloudPubSubHandler` tests --- tests/cloud/pub_sub/test_logging.py | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/tests/cloud/pub_sub/test_logging.py b/tests/cloud/pub_sub/test_logging.py index a94c17a8c..7b2f0f360 100644 --- a/tests/cloud/pub_sub/test_logging.py +++ b/tests/cloud/pub_sub/test_logging.py @@ -3,7 +3,7 @@ from logging import makeLogRecord from unittest.mock import patch -from octue.cloud.emulators._pub_sub import MESSAGES, MockService, MockSubscription, MockTopic +from octue.cloud.emulators._pub_sub import MESSAGES, MockService, MockTopic from octue.cloud.emulators.child import ServicePatcher from octue.cloud.pub_sub.logging import GoogleCloudPubSubHandler from octue.resources.service_backends import GCPPubSubBackend @@ -26,25 +26,16 @@ def setUpClass(cls): def test_emit(self): """Test the log message is published when `GoogleCloudPubSubHandler.emit` is called.""" - topic = MockTopic(name="octue.my-service.3-3-3", project_name="blah") - topic.create() - question_uuid = "96d69278-44ac-4631-aeea-c90fb08a1b2b" - subscription = MockSubscription(name=f"octue.my-service.3-3-3.answers.{question_uuid}", topic=topic) - subscription.create() - log_record = makeLogRecord({"msg": "Starting analysis."}) - backend = GCPPubSubBackend(project_name="blah") - with ServicePatcher(): - service = MockService(backend=backend) + service = MockService(backend=GCPPubSubBackend(project_name="blah")) GoogleCloudPubSubHandler( message_sender=service._send_message, - topic=topic, question_uuid=question_uuid, - originator=service.id, + originator="another/service:1.0.0", recipient="another/service:1.0.0", ).emit(log_record) @@ -57,9 +48,6 @@ def test_emit_with_non_json_serialisable_args(self): """Test that non-JSON-serialisable arguments to log messages are converted to their string representation before being serialised and published to the Pub/Sub topic. """ - topic = MockTopic(name="octue.my-service.3-3-4", project_name="blah") - topic.create() - non_json_serialisable_thing = NonJSONSerialisable() # Check that it can't be serialised to JSON. @@ -70,17 +58,14 @@ def test_emit_with_non_json_serialisable_args(self): {"msg": "%r is not JSON-serialisable but can go into a log message", "args": (non_json_serialisable_thing,)} ) - backend = GCPPubSubBackend(project_name="blah") - with ServicePatcher(): - service = MockService(backend=backend) + service = MockService(backend=GCPPubSubBackend(project_name="blah")) with patch("octue.cloud.emulators._pub_sub.MockPublisher.publish") as mock_publish: GoogleCloudPubSubHandler( message_sender=service._send_message, - topic=topic, question_uuid="question-uuid", - originator=service.id, + originator="another/service:1.0.0", recipient="another/service:1.0.0", ).emit(record) From c5075b4bceb724705ec22c0e1c0614b8b11f9bb8 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 9 Apr 2024 11:52:03 +0100 Subject: [PATCH 113/169] ENH: Make `Topic`'s representation consistent with `Subscription`'s --- octue/cloud/pub_sub/topic.py | 2 +- tests/cloud/pub_sub/test_topic.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/octue/cloud/pub_sub/topic.py b/octue/cloud/pub_sub/topic.py index 18c2947a4..42d33fafe 100644 --- a/octue/cloud/pub_sub/topic.py +++ b/octue/cloud/pub_sub/topic.py @@ -41,7 +41,7 @@ def __repr__(self): :return str: """ - return f"<{type(self).__name__}({self.name})>" + return f"<{type(self).__name__}(name={self.name!r})>" def create(self, allow_existing=False): """Create a Google Pub/Sub topic that can be published to. diff --git a/tests/cloud/pub_sub/test_topic.py b/tests/cloud/pub_sub/test_topic.py index 92802e51e..5267226da 100644 --- a/tests/cloud/pub_sub/test_topic.py +++ b/tests/cloud/pub_sub/test_topic.py @@ -10,7 +10,7 @@ class TestTopic(BaseTestCase): def test_repr(self): """Test that Topics are represented correctly.""" topic = Topic(name="world", project_name="my-project") - self.assertEqual(repr(topic), "") + self.assertEqual(repr(topic), "") def test_create(self): """Test that a topic can be created and that it's marked as having its creation triggered locally.""" From 2b96b82d0d71f2b81a8569fce9b25b8a92ed7f49 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 9 Apr 2024 11:59:05 +0100 Subject: [PATCH 114/169] TST: Update pub/sub event handler tests --- tests/cloud/pub_sub/test_events.py | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/tests/cloud/pub_sub/test_events.py b/tests/cloud/pub_sub/test_events.py index 92b7d43e8..dd11fe932 100644 --- a/tests/cloud/pub_sub/test_events.py +++ b/tests/cloud/pub_sub/test_events.py @@ -12,9 +12,9 @@ def create_mock_topic_and_subscription(): - """Create a question UUID, mock topic, and mock subscription. + """Create a question UUID, mock topic, mock subscription, and parent service. - :return (str, octue.cloud.emulators._pub_sub.MockTopic, octue.cloud.emulators._pub_sub.MockSubscription): question UUID, topic, and subscription + :return (str, octue.cloud.emulators._pub_sub.MockTopic, octue.cloud.emulators._pub_sub.MockSubscription, octue.cloud.emulators._pub_sub.MockService): question UUID, topic, subscription, and parent service """ question_uuid = str(uuid.uuid4()) @@ -26,7 +26,8 @@ def create_mock_topic_and_subscription(): with ServicePatcher(): parent = MockService( - service_id="my-org/my-service:1.0.0", backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME) + service_id="my-org/my-service:1.0.0", + backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME), ) return question_uuid, topic, subscription, parent @@ -73,7 +74,6 @@ def test_in_order_messages_are_handled_in_order(self): child._send_message( message=message["event"], attributes=message["attributes"], - topic=mock_topic, originator=parent.id, recipient=parent.id, ) @@ -124,7 +124,6 @@ def test_out_of_order_messages_are_handled_in_order(self): child._send_message( message=message["event"], attributes=message["attributes"], - topic=mock_topic, originator=parent.id, recipient=parent.id, ) @@ -183,7 +182,6 @@ def test_out_of_order_messages_with_end_message_first_are_handled_in_order(self) child._send_message( message=message["event"], attributes=message["attributes"], - topic=mock_topic, originator=parent.id, recipient=parent.id, ) @@ -235,7 +233,6 @@ def test_no_timeout(self): child._send_message( message=message["event"], attributes=message["attributes"], - topic=mock_topic, originator=parent.id, recipient=parent.id, ) @@ -271,7 +268,6 @@ def test_delivery_acknowledgement(self): child._send_message( message=message["event"], attributes=message["attributes"], - topic=mock_topic, originator=parent.id, recipient=parent.id, ) @@ -334,7 +330,6 @@ def test_error_not_raised_if_heartbeat_has_been_received_in_maximum_allowed_inte child._send_message( message=message["event"], attributes=message["attributes"], - topic=mock_topic, originator=parent.id, recipient=parent.id, ) @@ -411,7 +406,6 @@ def test_missing_messages_at_start_can_be_skipped(self): child._send_message( message=message["event"], attributes=message["attributes"], - topic=mock_topic, originator=parent.id, recipient=parent.id, ) @@ -464,7 +458,6 @@ def test_missing_messages_in_middle_can_skipped(self): child._send_message( message=message["event"], attributes=message["attributes"], - topic=mock_topic, originator=parent.id, recipient=parent.id, ) @@ -476,7 +469,6 @@ def test_missing_messages_in_middle_can_skipped(self): child._send_message( message={"kind": "finish-test", "order": 5}, attributes={"question_uuid": question_uuid, "sender_type": "CHILD"}, - topic=mock_topic, originator=parent.id, recipient=parent.id, ) @@ -529,7 +521,6 @@ def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): child._send_message( message=message["event"], attributes=message["attributes"], - topic=mock_topic, originator=parent.id, recipient=parent.id, ) @@ -541,7 +532,6 @@ def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): child._send_message( message={"kind": "test", "order": 5}, attributes={"order": 5, "question_uuid": question_uuid, "sender_type": "CHILD"}, - topic=mock_topic, originator=parent.id, recipient=parent.id, ) @@ -573,7 +563,6 @@ def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): child._send_message( message=message["event"], attributes=message["attributes"], - topic=mock_topic, originator=parent.id, recipient=parent.id, ) @@ -617,7 +606,6 @@ def test_all_messages_missing_apart_from_result(self): child._send_message( message={"kind": "finish-test", "order": 1000}, attributes={"question_uuid": question_uuid, "sender_type": "CHILD"}, - topic=mock_topic, originator=parent.id, recipient=parent.id, ) From f221b8e5bb8e1b452a471ceb0c62b27c5da68133 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 9 Apr 2024 12:10:05 +0100 Subject: [PATCH 115/169] TST: Simplify pub/sub event handler tests --- tests/cloud/pub_sub/test_events.py | 293 ++++++++++++++--------------- 1 file changed, 139 insertions(+), 154 deletions(-) diff --git a/tests/cloud/pub_sub/test_events.py b/tests/cloud/pub_sub/test_events.py index dd11fe932..4a5c231b0 100644 --- a/tests/cloud/pub_sub/test_events.py +++ b/tests/cloud/pub_sub/test_events.py @@ -11,37 +11,32 @@ from tests.base import BaseTestCase -def create_mock_topic_and_subscription(): - """Create a question UUID, mock topic, mock subscription, and parent service. - - :return (str, octue.cloud.emulators._pub_sub.MockTopic, octue.cloud.emulators._pub_sub.MockSubscription, octue.cloud.emulators._pub_sub.MockService): question UUID, topic, subscription, and parent service - """ - question_uuid = str(uuid.uuid4()) - - topic = MockTopic(name="octue.services", project_name=TEST_PROJECT_NAME) - topic.create(allow_existing=True) +class TestPubSubEventHandler(BaseTestCase): + @classmethod + def setUpClass(cls): + cls.question_uuid = str(uuid.uuid4()) - subscription = MockSubscription(name=f"my-org.my-service.1-0-0.answers.{question_uuid}", topic=topic) - subscription.create() + cls.topic = MockTopic(name="octue.services", project_name=TEST_PROJECT_NAME) + cls.topic.create(allow_existing=True) - with ServicePatcher(): - parent = MockService( - service_id="my-org/my-service:1.0.0", - backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME), + cls.subscription = MockSubscription( + name=f"my-org.my-service.1-0-0.answers.{cls.question_uuid}", + topic=cls.topic, ) + cls.subscription.create() - return question_uuid, topic, subscription, parent - + with ServicePatcher(): + cls.parent = MockService( + service_id="my-org/my-service:1.0.0", + backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME), + ) -class TestPubSubEventHandler(BaseTestCase): def test_timeout(self): """Test that a TimeoutError is raised if message handling takes longer than the given timeout.""" - question_uuid, _, mock_subscription, parent = create_mock_topic_and_subscription() - with ServicePatcher(): event_handler = GoogleCloudPubSubEventHandler( - subscription=mock_subscription, - receiving_service=parent, + subscription=self.subscription, + receiving_service=self.parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: message}, schema={}, ) @@ -51,12 +46,10 @@ def test_timeout(self): def test_in_order_messages_are_handled_in_order(self): """Test that messages received in order are handled in order.""" - question_uuid, mock_topic, mock_subscription, parent = create_mock_topic_and_subscription() - with ServicePatcher(): event_handler = GoogleCloudPubSubEventHandler( - subscription=mock_subscription, - receiving_service=parent, + subscription=self.subscription, + receiving_service=self.parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, schema={}, ) @@ -64,18 +57,21 @@ def test_in_order_messages_are_handled_in_order(self): child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) messages = [ - {"event": {"kind": "test"}, "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}}, - {"event": {"kind": "test"}, "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}}, - {"event": {"kind": "test"}, "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}}, - {"event": {"kind": "finish-test"}, "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}}, + {"event": {"kind": "test"}, "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}}, + {"event": {"kind": "test"}, "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}}, + {"event": {"kind": "test"}, "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}}, + { + "event": {"kind": "finish-test"}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, + }, ] for message in messages: child._send_message( message=message["event"], attributes=message["attributes"], - originator=parent.id, - recipient=parent.id, + originator=self.parent.id, + recipient=self.parent.id, ) result = event_handler.handle_events() @@ -88,12 +84,10 @@ def test_in_order_messages_are_handled_in_order(self): def test_out_of_order_messages_are_handled_in_order(self): """Test that messages received out of order are handled in order.""" - question_uuid, mock_topic, mock_subscription, parent = create_mock_topic_and_subscription() - with ServicePatcher(): event_handler = GoogleCloudPubSubEventHandler( - subscription=mock_subscription, - receiving_service=parent, + subscription=self.subscription, + receiving_service=self.parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, schema={}, ) @@ -103,19 +97,19 @@ def test_out_of_order_messages_are_handled_in_order(self): messages = [ { "event": {"kind": "test", "order": 1}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, }, { "event": {"kind": "test", "order": 2}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, }, { "event": {"kind": "test", "order": 0}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, }, { "event": {"kind": "finish-test", "order": 3}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, }, ] @@ -124,8 +118,8 @@ def test_out_of_order_messages_are_handled_in_order(self): child._send_message( message=message["event"], attributes=message["attributes"], - originator=parent.id, - recipient=parent.id, + originator=self.parent.id, + recipient=self.parent.id, ) result = event_handler.handle_events() @@ -146,12 +140,10 @@ def test_out_of_order_messages_with_end_message_first_are_handled_in_order(self) """Test that messages received out of order and with the final message (the message that triggers a value to be returned) are handled in order. """ - question_uuid, mock_topic, mock_subscription, parent = create_mock_topic_and_subscription() - with ServicePatcher(): event_handler = GoogleCloudPubSubEventHandler( - subscription=mock_subscription, - receiving_service=parent, + subscription=self.subscription, + receiving_service=self.parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, schema={}, ) @@ -161,19 +153,19 @@ def test_out_of_order_messages_with_end_message_first_are_handled_in_order(self) messages = [ { "event": {"kind": "finish-test", "order": 3}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, }, { "event": {"kind": "test", "order": 1}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, }, { "event": {"kind": "test", "order": 2}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, }, { "event": {"kind": "test", "order": 0}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, }, ] @@ -182,8 +174,8 @@ def test_out_of_order_messages_with_end_message_first_are_handled_in_order(self) child._send_message( message=message["event"], attributes=message["attributes"], - originator=parent.id, - recipient=parent.id, + originator=self.parent.id, + recipient=self.parent.id, ) result = event_handler.handle_events() @@ -202,12 +194,10 @@ def test_out_of_order_messages_with_end_message_first_are_handled_in_order(self) def test_no_timeout(self): """Test that message handling works with no timeout.""" - question_uuid, mock_topic, mock_subscription, parent = create_mock_topic_and_subscription() - with ServicePatcher(): event_handler = GoogleCloudPubSubEventHandler( - subscription=mock_subscription, - receiving_service=parent, + subscription=self.subscription, + receiving_service=self.parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, schema={}, ) @@ -217,15 +207,15 @@ def test_no_timeout(self): messages = [ { "event": {"kind": "test", "order": 0}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, }, { "event": {"kind": "test", "order": 1}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, }, { "event": {"kind": "finish-test", "order": 2}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, }, ] @@ -233,8 +223,8 @@ def test_no_timeout(self): child._send_message( message=message["event"], attributes=message["attributes"], - originator=parent.id, - recipient=parent.id, + originator=self.parent.id, + recipient=self.parent.id, ) result = event_handler.handle_events(timeout=None) @@ -247,20 +237,18 @@ def test_no_timeout(self): def test_delivery_acknowledgement(self): """Test that a delivery acknowledgement message is handled correctly.""" - question_uuid, mock_topic, mock_subscription, parent = create_mock_topic_and_subscription() - with ServicePatcher(): - event_handler = GoogleCloudPubSubEventHandler(subscription=mock_subscription, receiving_service=parent) + event_handler = GoogleCloudPubSubEventHandler(subscription=self.subscription, receiving_service=self.parent) child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) messages = [ { "event": {"kind": "delivery_acknowledgement", "datetime": datetime.datetime.utcnow().isoformat()}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, }, { "event": {"kind": "result"}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, }, ] @@ -268,8 +256,8 @@ def test_delivery_acknowledgement(self): child._send_message( message=message["event"], attributes=message["attributes"], - originator=parent.id, - recipient=parent.id, + originator=self.parent.id, + recipient=self.parent.id, ) result = event_handler.handle_events() @@ -277,10 +265,8 @@ def test_delivery_acknowledgement(self): def test_error_raised_if_heartbeat_not_received_before_checked(self): """Test that an error is raised if a heartbeat isn't received before a heartbeat is first checked for.""" - question_uuid, _, mock_subscription, parent = create_mock_topic_and_subscription() - with ServicePatcher(): - event_handler = GoogleCloudPubSubEventHandler(subscription=mock_subscription, receiving_service=parent) + event_handler = GoogleCloudPubSubEventHandler(subscription=self.subscription, receiving_service=self.parent) with self.assertRaises(TimeoutError) as error: event_handler.handle_events(maximum_heartbeat_interval=0) @@ -290,10 +276,8 @@ def test_error_raised_if_heartbeat_not_received_before_checked(self): def test_error_raised_if_heartbeats_stop_being_received(self): """Test that an error is raised if heartbeats stop being received within the maximum interval.""" - question_uuid, _, mock_subscription, parent = create_mock_topic_and_subscription() - with ServicePatcher(): - event_handler = GoogleCloudPubSubEventHandler(subscription=mock_subscription, receiving_service=parent) + event_handler = GoogleCloudPubSubEventHandler(subscription=self.subscription, receiving_service=self.parent) event_handler._last_heartbeat = datetime.datetime.now() - datetime.timedelta(seconds=30) @@ -304,10 +288,8 @@ def test_error_raised_if_heartbeats_stop_being_received(self): def test_error_not_raised_if_heartbeat_has_been_received_in_maximum_allowed_interval(self): """Test that an error is not raised if a heartbeat has been received in the maximum allowed interval.""" - question_uuid, mock_topic, mock_subscription, parent = create_mock_topic_and_subscription() - with ServicePatcher(): - event_handler = GoogleCloudPubSubEventHandler(subscription=mock_subscription, receiving_service=parent) + event_handler = GoogleCloudPubSubEventHandler(subscription=self.subscription, receiving_service=self.parent) child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) event_handler._last_heartbeat = datetime.datetime.now() @@ -318,11 +300,11 @@ def test_error_not_raised_if_heartbeat_has_been_received_in_maximum_allowed_inte "kind": "delivery_acknowledgement", "datetime": datetime.datetime.utcnow().isoformat(), }, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, }, { "event": {"kind": "result"}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, }, ] @@ -330,8 +312,8 @@ def test_error_not_raised_if_heartbeat_has_been_received_in_maximum_allowed_inte child._send_message( message=message["event"], attributes=message["attributes"], - originator=parent.id, - recipient=parent.id, + originator=self.parent.id, + recipient=self.parent.id, ) with patch( @@ -342,10 +324,8 @@ def test_error_not_raised_if_heartbeat_has_been_received_in_maximum_allowed_inte def test_time_since_last_heartbeat_is_none_if_no_heartbeat_received_yet(self): """Test that the time since the last heartbeat is `None` if no heartbeat has been received yet.""" - question_uuid, _, mock_subscription, parent = create_mock_topic_and_subscription() - with ServicePatcher(): - event_handler = GoogleCloudPubSubEventHandler(subscription=mock_subscription, receiving_service=parent) + event_handler = GoogleCloudPubSubEventHandler(subscription=self.subscription, receiving_service=self.parent) self.assertIsNone(event_handler._time_since_last_heartbeat) @@ -353,26 +333,22 @@ def test_total_run_time_is_none_if_handle_events_has_not_been_called(self): """Test that the total run time for the message handler is `None` if the `handle_events` method has not been called. """ - question_uuid, _, mock_subscription, parent = create_mock_topic_and_subscription() - event_handler = GoogleCloudPubSubEventHandler(subscription=mock_subscription, receiving_service=parent) + event_handler = GoogleCloudPubSubEventHandler(subscription=self.subscription, receiving_service=self.parent) self.assertIsNone(event_handler.total_run_time) def test_time_since_missing_message_is_none_if_no_unhandled_missing_messages(self): """Test that the `time_since_missing_message` property is `None` if there are no unhandled missing messages.""" - question_uuid, _, mock_subscription, parent = create_mock_topic_and_subscription() - event_handler = GoogleCloudPubSubEventHandler(subscription=mock_subscription, receiving_service=parent) + event_handler = GoogleCloudPubSubEventHandler(subscription=self.subscription, receiving_service=self.parent) self.assertIsNone(event_handler.time_since_missing_event) def test_missing_messages_at_start_can_be_skipped(self): """Test that missing messages at the start of the event stream can be skipped if they aren't received after a given time period if subsequent messages have been received. """ - question_uuid, mock_topic, mock_subscription, parent = create_mock_topic_and_subscription() - with ServicePatcher(): event_handler = GoogleCloudPubSubEventHandler( - subscription=mock_subscription, - receiving_service=parent, + subscription=self.subscription, + receiving_service=self.parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, schema={}, skip_missing_events_after=0, @@ -386,19 +362,19 @@ def test_missing_messages_at_start_can_be_skipped(self): messages = [ { "event": {"kind": "test", "order": 2}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, }, { "event": {"kind": "test", "order": 3}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, }, { "event": {"kind": "test", "order": 4}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, }, { "event": {"kind": "finish-test", "order": 5}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, }, ] @@ -406,8 +382,8 @@ def test_missing_messages_at_start_can_be_skipped(self): child._send_message( message=message["event"], attributes=message["attributes"], - originator=parent.id, - recipient=parent.id, + originator=self.parent.id, + recipient=self.parent.id, ) result = event_handler.handle_events() @@ -425,12 +401,10 @@ def test_missing_messages_at_start_can_be_skipped(self): def test_missing_messages_in_middle_can_skipped(self): """Test that missing messages in the middle of the event stream can be skipped.""" - question_uuid, mock_topic, mock_subscription, parent = create_mock_topic_and_subscription() - with ServicePatcher(): event_handler = GoogleCloudPubSubEventHandler( - subscription=mock_subscription, - receiving_service=parent, + subscription=self.subscription, + receiving_service=self.parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, schema={}, skip_missing_events_after=0, @@ -442,15 +416,15 @@ def test_missing_messages_in_middle_can_skipped(self): messages = [ { "event": {"kind": "test", "order": 0}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, }, { "event": {"kind": "test", "order": 1}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, }, { "event": {"kind": "test", "order": 2}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, }, ] @@ -458,8 +432,8 @@ def test_missing_messages_in_middle_can_skipped(self): child._send_message( message=message["event"], attributes=message["attributes"], - originator=parent.id, - recipient=parent.id, + originator=self.parent.id, + recipient=self.parent.id, ) # Simulate missing messages. @@ -468,9 +442,9 @@ def test_missing_messages_in_middle_can_skipped(self): # Send a final message. child._send_message( message={"kind": "finish-test", "order": 5}, - attributes={"question_uuid": question_uuid, "sender_type": "CHILD"}, - originator=parent.id, - recipient=parent.id, + attributes={"question_uuid": self.question_uuid, "sender_type": "CHILD"}, + originator=self.parent.id, + recipient=self.parent.id, ) event_handler.handle_events() @@ -488,12 +462,10 @@ def test_missing_messages_in_middle_can_skipped(self): def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): """Test that multiple blocks of missing messages in the middle of the event stream can be skipped.""" - question_uuid, mock_topic, mock_subscription, parent = create_mock_topic_and_subscription() - with ServicePatcher(): event_handler = GoogleCloudPubSubEventHandler( - subscription=mock_subscription, - receiving_service=parent, + subscription=self.subscription, + receiving_service=self.parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, schema={}, skip_missing_events_after=0, @@ -505,15 +477,15 @@ def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): messages = [ { "event": {"kind": "test", "order": 0}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, }, { "event": {"kind": "test", "order": 1}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, }, { "event": {"kind": "test", "order": 2}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, }, ] @@ -521,8 +493,8 @@ def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): child._send_message( message=message["event"], attributes=message["attributes"], - originator=parent.id, - recipient=parent.id, + originator=self.parent.id, + recipient=self.parent.id, ) # Simulate missing messages. @@ -531,9 +503,9 @@ def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): # Send another message. child._send_message( message={"kind": "test", "order": 5}, - attributes={"order": 5, "question_uuid": question_uuid, "sender_type": "CHILD"}, - originator=parent.id, - recipient=parent.id, + attributes={"order": 5, "question_uuid": self.question_uuid, "sender_type": "CHILD"}, + originator=self.parent.id, + recipient=self.parent.id, ) # Simulate more missing messages. @@ -543,19 +515,19 @@ def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): messages = [ { "event": {"kind": "test", "order": 20}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, }, { "event": {"kind": "test", "order": 21}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, }, { "event": {"kind": "test", "order": 22}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, }, { "event": {"kind": "finish-test", "order": 23}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, }, ] @@ -563,8 +535,8 @@ def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): child._send_message( message=message["event"], attributes=message["attributes"], - originator=parent.id, - recipient=parent.id, + originator=self.parent.id, + recipient=self.parent.id, ) event_handler.handle_events() @@ -586,12 +558,10 @@ def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): def test_all_messages_missing_apart_from_result(self): """Test that the result message is still handled if all other messages are missing.""" - question_uuid, mock_topic, mock_subscription, parent = create_mock_topic_and_subscription() - with ServicePatcher(): event_handler = GoogleCloudPubSubEventHandler( - subscription=mock_subscription, - receiving_service=parent, + subscription=self.subscription, + receiving_service=self.parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, schema={}, skip_missing_events_after=0, @@ -605,9 +575,9 @@ def test_all_messages_missing_apart_from_result(self): # Send the result message. child._send_message( message={"kind": "finish-test", "order": 1000}, - attributes={"question_uuid": question_uuid, "sender_type": "CHILD"}, - originator=parent.id, - recipient=parent.id, + attributes={"question_uuid": self.question_uuid, "sender_type": "CHILD"}, + originator=self.parent.id, + recipient=self.parent.id, ) event_handler.handle_events() @@ -617,24 +587,41 @@ def test_all_messages_missing_apart_from_result(self): class TestPullAndEnqueueAvailableMessages(BaseTestCase): + @classmethod + def setUpClass(cls): + cls.question_uuid = str(uuid.uuid4()) + + cls.topic = MockTopic(name="octue.services", project_name=TEST_PROJECT_NAME) + cls.topic.create(allow_existing=True) + + cls.subscription = MockSubscription( + name=f"my-org.my-service.1-0-0.answers.{cls.question_uuid}", + topic=cls.topic, + ) + cls.subscription.create() + + with ServicePatcher(): + cls.parent = MockService( + service_id="my-org/my-service:1.0.0", + backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME), + ) + def test_pull_and_enqueue_available_events(self): """Test that pulling and enqueuing a message works.""" - question_uuid, mock_topic, _, parent = create_mock_topic_and_subscription() - with ServicePatcher(): - mock_subscription = MockSubscription( - name=f"my-org.my-service.1-0-0.answers.{question_uuid}", - topic=mock_topic, + self.subscription = MockSubscription( + name=f"my-org.my-service.1-0-0.answers.{self.question_uuid}", + topic=self.topic, ) event_handler = GoogleCloudPubSubEventHandler( - subscription=mock_subscription, - receiving_service=parent, + subscription=self.subscription, + receiving_service=self.parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, schema={}, ) - event_handler.question_uuid = question_uuid + event_handler.question_uuid = self.question_uuid event_handler.child_sruid = "my-org/my-service:1.0.0" event_handler._child_sdk_version = "0.1.3" event_handler.waiting_events = {} @@ -642,14 +629,14 @@ def test_pull_and_enqueue_available_events(self): # Enqueue a mock message for a mock subscription to receive. mock_message = {"kind": "test"} - MESSAGES[question_uuid] = [ + MESSAGES[self.question_uuid] = [ MockMessage.from_primitive( mock_message, attributes={ "order": 0, - "question_uuid": question_uuid, - "originator": parent.id, - "sender": parent.id, + "question_uuid": self.question_uuid, + "originator": self.parent.id, + "sender": self.parent.id, "sender_type": "CHILD", "sender_sdk_version": "0.50.0", "recipient": "my-org/my-service:1.0.0", @@ -663,17 +650,15 @@ def test_pull_and_enqueue_available_events(self): def test_timeout_error_raised_if_result_message_not_received_in_time(self): """Test that a timeout error is raised if a result message is not received in time.""" - question_uuid, mock_topic, _, parent = create_mock_topic_and_subscription() - with ServicePatcher(): - mock_subscription = MockSubscription( - name=f"my-org.my-service.1-0-0.answers.{question_uuid}", - topic=mock_topic, + self.subscription = MockSubscription( + name=f"my-org.my-service.1-0-0.answers.{self.question_uuid}", + topic=self.topic, ) event_handler = GoogleCloudPubSubEventHandler( - subscription=mock_subscription, - receiving_service=parent, + subscription=self.subscription, + receiving_service=self.parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, ) From c77f7f45b7294b994cdbfac32945d69ab06d7a5d Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 9 Apr 2024 12:36:13 +0100 Subject: [PATCH 116/169] TST: Factor out patching to start of services test class --- octue/utils/patches.py | 16 +- tests/cloud/pub_sub/test_service.py | 579 +++++++++++++--------------- 2 files changed, 287 insertions(+), 308 deletions(-) diff --git a/octue/utils/patches.py b/octue/utils/patches.py index 54883acff..b2196b62c 100644 --- a/octue/utils/patches.py +++ b/octue/utils/patches.py @@ -14,11 +14,25 @@ def __enter__(self): :return list(unittest.mock.MagicMock): """ - return [patch.start() for patch in self.patches] + return self.start() def __exit__(self, *args, **kwargs): """Stop the patches. + :return None: + """ + self.stop() + + def start(self): + """Start the patches and return the mocks they produce. + + :return list(unittest.mock.MagicMock): + """ + return [patch.start() for patch in self.patches] + + def stop(self): + """Stop the patches. + :return None: """ for patch in self.patches: diff --git a/tests/cloud/pub_sub/test_service.py b/tests/cloud/pub_sub/test_service.py index 2133b1399..f1ac55b20 100644 --- a/tests/cloud/pub_sub/test_service.py +++ b/tests/cloud/pub_sub/test_service.py @@ -44,23 +44,30 @@ class TestService(BaseTestCase): @classmethod def setUpClass(cls): + """Start the service patcher and create a mock services topic. + + :return None: + """ + cls.service_patcher.start() topic = MockTopic(name="octue.services", project_name=TEST_PROJECT_NAME) + topic.create(allow_existing=True) + + @classmethod + def tearDownClass(cls): + """Stop the services patcher. - with cls.service_patcher: - topic.create(allow_existing=True) + :return None: + """ + cls.service_patcher.stop() def test_repr(self): """Test that services are represented as a string correctly.""" - with self.service_patcher: - service = Service(backend=BACKEND) - + service = Service(backend=BACKEND) self.assertEqual(repr(service), f"") def test_repr_with_name(self): """Test that services are represented using their name if they have one.""" - with self.service_patcher: - service = Service(backend=BACKEND, name=f"octue/blah-service:{MOCK_SERVICE_REVISION_TAG}") - + service = Service(backend=BACKEND, name=f"octue/blah-service:{MOCK_SERVICE_REVISION_TAG}") self.assertEqual(repr(service), f"") def test_service_id_cannot_be_non_none_empty_value(self): @@ -76,26 +83,21 @@ def test_service_id_cannot_be_non_none_empty_value(self): def test_serve_fails_if_service_with_same_id_already_exists(self): """Test that serving a service fails if a service with the same name already exists.""" - with self.service_patcher: - with patch( - "octue.cloud.pub_sub.service.Topic.create", - side_effect=google.api_core.exceptions.AlreadyExists(""), - ): - with self.assertRaises(exceptions.ServiceAlreadyExists): - MockService( - backend=BACKEND, - service_id=f"my-org/existing-service:{MOCK_SERVICE_REVISION_TAG}", - ).serve() + with patch( + "octue.cloud.pub_sub.service.Subscription.create", + side_effect=google.api_core.exceptions.AlreadyExists(""), + ): + with self.assertRaises(exceptions.ServiceAlreadyExists): + MockService(backend=BACKEND, service_id=f"my-org/existing-service:{MOCK_SERVICE_REVISION_TAG}").serve() def test_serve(self): """Test that serving works with a unique service ID. Test that the returned future has itself been returned and that the returned subscriber has been closed. """ - with self.service_patcher: - future, subscriber = MockService( - backend=BACKEND, - service_id=f"my-org/existing-service:{MOCK_SERVICE_REVISION_TAG}", - ).serve() + future, subscriber = MockService( + backend=BACKEND, + service_id=f"my-org/existing-service:{MOCK_SERVICE_REVISION_TAG}", + ).serve() self.assertFalse(future.cancelled) self.assertTrue(future.returned) @@ -105,9 +107,8 @@ def test_serve_detached(self): """Test that, when serving a service in detached mode, the returned future is not cancelled or returned and that the returned subscriber is not closed. """ - with self.service_patcher: - service = MockService(backend=BACKEND, service_id=f"my-org/existing-service-d:{MOCK_SERVICE_REVISION_TAG}") - future, subscriber = service.serve(detach=True) + service = MockService(backend=BACKEND, service_id=f"my-org/existing-service-d:{MOCK_SERVICE_REVISION_TAG}") + future, subscriber = service.serve(detach=True) self.assertFalse(future.cancelled) self.assertFalse(future.returned) @@ -117,68 +118,63 @@ def test_ask_unregistered_service_revision_when_service_registries_specified_res """Test that an error is raised if attempting to ask an unregistered service a question when service registries are being used. """ - with patch("octue.cloud.pub_sub.service.Topic", new=MockTopic): - service = MockService( - backend=BACKEND, - service_registries=[{"name": "Octue Registry", "endpoint": "https://blah.com/services"}], - ) + service = MockService( + backend=BACKEND, + service_registries=[{"name": "Octue Registry", "endpoint": "https://blah.com/services"}], + ) - mock_response = requests.Response() - mock_response.status_code = 404 + mock_response = requests.Response() + mock_response.status_code = 404 - with patch("requests.get", return_value=mock_response): - with self.assertRaises(exceptions.ServiceNotFound): - service.ask( - service_id=f"my-org/unregistered-service:{MOCK_SERVICE_REVISION_TAG}", - input_values=[1, 2, 3, 4], - ) + with patch("requests.get", return_value=mock_response): + with self.assertRaises(exceptions.ServiceNotFound): + service.ask( + service_id=f"my-org/unregistered-service:{MOCK_SERVICE_REVISION_TAG}", + input_values=[1, 2, 3, 4], + ) def test_ask_unregistered_service_with_no_revision_tag_when_service_registries_specified_results_in_error(self): """Test that an error is raised when attempting to ask a question to an unregistered service without including revision tag when service registries are being used. """ - with patch("octue.cloud.pub_sub.service.Topic", new=MockTopic): - service = MockService( - backend=BACKEND, - service_registries=[{"name": "Octue Registry", "endpoint": "https://blah.com/services"}], - ) + service = MockService( + backend=BACKEND, + service_registries=[{"name": "Octue Registry", "endpoint": "https://blah.com/services"}], + ) - mock_response = requests.Response() - mock_response.status_code = 404 + mock_response = requests.Response() + mock_response.status_code = 404 - with patch("requests.get", return_value=mock_response): - with self.assertRaises(exceptions.ServiceNotFound): - service.ask(service_id="my-org/unregistered-service", input_values=[1, 2, 3, 4]) + with patch("requests.get", return_value=mock_response): + with self.assertRaises(exceptions.ServiceNotFound): + service.ask(service_id="my-org/unregistered-service", input_values=[1, 2, 3, 4]) def test_ask_service_with_no_revision_tag_when_service_registries_not_specified_results_in_error(self): """Test that an error is raised when attempting to ask a question to a service without including a revision tag when service registries are not being used. """ - with patch("octue.cloud.pub_sub.service.Topic", new=MockTopic): - service = MockService(backend=BACKEND) + service = MockService(backend=BACKEND) - mock_response = requests.Response() - mock_response.status_code = 404 + mock_response = requests.Response() + mock_response.status_code = 404 - with patch("requests.get", return_value=mock_response): - with self.assertRaises(exceptions.InvalidServiceID): - service.ask(service_id="my-org/unregistered-service", input_values=[1, 2, 3, 4]) + with patch("requests.get", return_value=mock_response): + with self.assertRaises(exceptions.InvalidServiceID): + service.ask(service_id="my-org/unregistered-service", input_values=[1, 2, 3, 4]) def test_timeout_error_raised_if_no_messages_received_when_waiting(self): """Test that a TimeoutError is raised if no messages are received while waiting.""" mock_topic = MockTopic(name="amazing.service.9-9-9", project_name=TEST_PROJECT_NAME) mock_subscription = MockSubscription(name="amazing.service.9-9-9", topic=mock_topic) - with self.service_patcher: - service = Service(backend=BACKEND) + service = Service(backend=BACKEND) - with self.assertRaises(TimeoutError): - service.wait_for_answer(subscription=mock_subscription, timeout=0.01) + with self.assertRaises(TimeoutError): + service.wait_for_answer(subscription=mock_subscription, timeout=0.01) def test_error_raised_if_attempting_to_wait_for_answer_from_non_pull_subscription(self): """Test that an error is raised if attempting to wait for an answer from a push subscription.""" - with self.service_patcher: - service = Service(backend=BACKEND) + service = Service(backend=BACKEND) subscription = MockSubscription( name="world", @@ -191,18 +187,17 @@ def test_error_raised_if_attempting_to_wait_for_answer_from_non_pull_subscriptio def test_exceptions_in_responder_are_handled_and_sent_to_asker(self): """Test that exceptions raised in the child service are handled and sent back to the asker.""" - with self.service_patcher: - child = self.make_new_child_with_error( - twined.exceptions.InvalidManifestContents("'met_mast_id' is a required property") - ) + child = self.make_new_child_with_error( + twined.exceptions.InvalidManifestContents("'met_mast_id' is a required property") + ) - parent = MockService(backend=BACKEND, children={child.id: child}) + parent = MockService(backend=BACKEND, children={child.id: child}) - child.serve() - subscription, _ = parent.ask(service_id=child.id, input_values={}) + child.serve() + subscription, _ = parent.ask(service_id=child.id, input_values={}) - with self.assertRaises(twined.exceptions.InvalidManifestContents) as context: - parent.wait_for_answer(subscription=subscription) + with self.assertRaises(twined.exceptions.InvalidManifestContents) as context: + parent.wait_for_answer(subscription=subscription) self.assertIn("'met_mast_id' is a required property", context.exception.args[0]) @@ -210,15 +205,14 @@ def test_exceptions_with_multiple_arguments_in_responder_are_handled_and_sent_to """Test that exceptions with multiple arguments raised in the child service are handled and sent back to the asker. """ - with self.service_patcher: - child = self.make_new_child_with_error(FileNotFoundError(2, "No such file or directory: 'blah'")) - parent = MockService(backend=BACKEND, children={child.id: child}) + child = self.make_new_child_with_error(FileNotFoundError(2, "No such file or directory: 'blah'")) + parent = MockService(backend=BACKEND, children={child.id: child}) - child.serve() - subscription, _ = parent.ask(service_id=child.id, input_values={}) + child.serve() + subscription, _ = parent.ask(service_id=child.id, input_values={}) - with self.assertRaises(FileNotFoundError) as context: - parent.wait_for_answer(subscription) + with self.assertRaises(FileNotFoundError) as context: + parent.wait_for_answer(subscription) self.assertIn("[Errno 2] No such file or directory: 'blah'", format(context.exception)) @@ -228,15 +222,14 @@ def test_unknown_exceptions_in_responder_are_handled_and_sent_to_asker(self): class AnUnknownException(Exception): pass - with self.service_patcher: - child = self.make_new_child_with_error(AnUnknownException("This is an exception unknown to the asker.")) - parent = MockService(backend=BACKEND, children={child.id: child}) + child = self.make_new_child_with_error(AnUnknownException("This is an exception unknown to the asker.")) + parent = MockService(backend=BACKEND, children={child.id: child}) - child.serve() - subscription, _ = parent.ask(service_id=child.id, input_values={}) + child.serve() + subscription, _ = parent.ask(service_id=child.id, input_values={}) - with self.assertRaises(Exception) as context: - parent.wait_for_answer(subscription) + with self.assertRaises(Exception) as context: + parent.wait_for_answer(subscription) self.assertEqual(type(context.exception).__name__, "AnUnknownException") self.assertIn("This is an exception unknown to the asker.", context.exception.args[0]) @@ -246,18 +239,13 @@ def test_ask_with_real_run_function_with_no_log_message_forwarding(self): run function rather than a mock so that the underlying `Runner` instance is used, and check that remote log messages aren't forwarded to the local logger. """ - with self.service_patcher: - child = MockService( - backend=BACKEND, - service_id="truly/madly:deeply", - run_function=self.create_run_function(), - ) - parent = MockService(backend=BACKEND, children={child.id: child}) + child = MockService(backend=BACKEND, service_id="truly/madly:deeply", run_function=self.create_run_function()) + parent = MockService(backend=BACKEND, children={child.id: child}) - with self.assertLogs() as logging_context: - child.serve() - subscription, _ = parent.ask(service_id=child.id, input_values={}, subscribe_to_logs=False) - answer = parent.wait_for_answer(subscription) + with self.assertLogs() as logging_context: + child.serve() + subscription, _ = parent.ask(service_id=child.id, input_values={}, subscribe_to_logs=False) + answer = parent.wait_for_answer(subscription) self.assertEqual( answer, @@ -271,19 +259,18 @@ def test_ask_with_real_run_function_with_log_message_forwarding(self): run function rather than a mock so that the underlying `Runner` instance is used, and check that remote log messages are forwarded to the local logger. """ - with self.service_patcher: - child = MockService( - backend=BACKEND, - service_id="my-super/service:6.0.1", - run_function=self.create_run_function(), - ) + child = MockService( + backend=BACKEND, + service_id="my-super/service:6.0.1", + run_function=self.create_run_function(), + ) - parent = MockService(backend=BACKEND, children={child.id: child}) + parent = MockService(backend=BACKEND, children={child.id: child}) - with self.assertLogs() as logs_context_manager: - child.serve() - subscription, _ = parent.ask(service_id=child.id, input_values={}, subscribe_to_logs=True) - answer = parent.wait_for_answer(subscription) + with self.assertLogs() as logs_context_manager: + child.serve() + subscription, _ = parent.ask(service_id=child.id, input_values={}, subscribe_to_logs=True) + answer = parent.wait_for_answer(subscription) self.assertEqual( answer, @@ -322,14 +309,13 @@ def mock_app(analysis): return Runner(app_src=mock_app, twine='{"input_values_schema": {"type": "object", "required": []}}').run - with self.service_patcher: - child = MockService(backend=BACKEND, run_function=create_exception_logging_run_function()) - parent = MockService(backend=BACKEND, children={child.id: child}) + child = MockService(backend=BACKEND, run_function=create_exception_logging_run_function()) + parent = MockService(backend=BACKEND, children={child.id: child}) - with self.assertLogs(level=logging.ERROR) as logs_context_manager: - child.serve() - subscription, _ = parent.ask(service_id=child.id, input_values={}, subscribe_to_logs=True) - parent.wait_for_answer(subscription, timeout=100) + with self.assertLogs(level=logging.ERROR) as logs_context_manager: + child.serve() + subscription, _ = parent.ask(service_id=child.id, input_values={}, subscribe_to_logs=True) + parent.wait_for_answer(subscription, timeout=100) error_logged = False @@ -365,15 +351,14 @@ def mock_app(analysis): return Runner(app_src=mock_app, twine=twine).run - with self.service_patcher: - child = MockService(backend=BACKEND, run_function=create_run_function_with_monitoring()) - parent = MockService(backend=BACKEND, children={child.id: child}) + child = MockService(backend=BACKEND, run_function=create_run_function_with_monitoring()) + parent = MockService(backend=BACKEND, children={child.id: child}) - child.serve() - subscription, _ = parent.ask(child.id, input_values={}) + child.serve() + subscription, _ = parent.ask(child.id, input_values={}) - monitoring_data = [] - parent.wait_for_answer(subscription, handle_monitor_message=lambda data: monitoring_data.append(data)) + monitoring_data = [] + parent.wait_for_answer(subscription, handle_monitor_message=lambda data: monitoring_data.append(data)) self.assertEqual( monitoring_data, @@ -404,19 +389,15 @@ def mock_app(analysis): return Runner(app_src=mock_app, twine=twine).run - with self.service_patcher: - child = MockService(backend=BACKEND, run_function=create_run_function_with_monitoring()) - parent = MockService(backend=BACKEND, children={child.id: child}) + child = MockService(backend=BACKEND, run_function=create_run_function_with_monitoring()) + parent = MockService(backend=BACKEND, children={child.id: child}) - child.serve() - subscription, _ = parent.ask(child.id, input_values={}) - monitoring_data = [] + child.serve() + subscription, _ = parent.ask(child.id, input_values={}) + monitoring_data = [] - with self.assertRaises(InvalidMonitorMessage): - parent.wait_for_answer( - subscription, - handle_monitor_message=lambda data: monitoring_data.append(data), - ) + with self.assertRaises(InvalidMonitorMessage): + parent.wait_for_answer(subscription, handle_monitor_message=lambda data: monitoring_data.append(data)) self.assertEqual(monitoring_data, [{"status": "my first monitor message"}]) @@ -429,20 +410,18 @@ def test_ask_with_non_json_python_primitive_input_values(self): def run_function(analysis_id, input_values, *args, **kwargs): return MockAnalysis(output_values=input_values) - with self.service_patcher: - child = MockService(backend=BACKEND, run_function=lambda *args, **kwargs: run_function(*args, **kwargs)) - parent = MockService(backend=BACKEND, children={child.id: child}) - child.serve() - - subscription, _ = parent.ask( - service_id=child.id, - input_values=input_values, - subscribe_to_logs=True, - save_diagnostics="SAVE_DIAGNOSTICS_ON_CRASH", - ) + child = MockService(backend=BACKEND, run_function=lambda *args, **kwargs: run_function(*args, **kwargs)) + parent = MockService(backend=BACKEND, children={child.id: child}) + child.serve() - answer = parent.wait_for_answer(subscription) + subscription, _ = parent.ask( + service_id=child.id, + input_values=input_values, + subscribe_to_logs=True, + save_diagnostics="SAVE_DIAGNOSTICS_ON_CRASH", + ) + answer = parent.wait_for_answer(subscription) self.assertEqual(answer["output_values"], input_values) def test_ask_with_input_manifest(self): @@ -460,14 +439,13 @@ def test_ask_with_input_manifest(self): } ) - with self.service_patcher: - child = self.make_new_child(BACKEND, run_function_returnee=MockAnalysis()) - parent = MockService(backend=BACKEND, children={child.id: child}) - child.serve() + child = self.make_new_child(BACKEND, run_function_returnee=MockAnalysis()) + parent = MockService(backend=BACKEND, children={child.id: child}) + child.serve() - with patch("google.cloud.storage.blob.Blob.generate_signed_url", new=mock_generate_signed_url): - subscription, _ = parent.ask(service_id=child.id, input_values={}, input_manifest=input_manifest) - answer = parent.wait_for_answer(subscription) + with patch("google.cloud.storage.blob.Blob.generate_signed_url", new=mock_generate_signed_url): + subscription, _ = parent.ask(service_id=child.id, input_values={}, input_manifest=input_manifest) + answer = parent.wait_for_answer(subscription) self.assertEqual( answer, @@ -489,14 +467,13 @@ def test_ask_with_input_manifest_and_no_input_values(self): } ) - with self.service_patcher: - child = self.make_new_child(BACKEND, run_function_returnee=MockAnalysis()) - parent = MockService(backend=BACKEND, children={child.id: child}) - child.serve() + child = self.make_new_child(BACKEND, run_function_returnee=MockAnalysis()) + parent = MockService(backend=BACKEND, children={child.id: child}) + child.serve() - with patch("google.cloud.storage.blob.Blob.generate_signed_url", new=mock_generate_signed_url): - subscription, _ = parent.ask(service_id=child.id, input_manifest=input_manifest) - answer = parent.wait_for_answer(subscription) + with patch("google.cloud.storage.blob.Blob.generate_signed_url", new=mock_generate_signed_url): + subscription, _ = parent.ask(service_id=child.id, input_manifest=input_manifest) + answer = parent.wait_for_answer(subscription) self.assertEqual( answer, @@ -507,8 +484,7 @@ def test_ask_with_input_manifest_with_local_paths_raises_error(self): """Test that an error is raised if an input manifest whose datasets and/or files are not located in the cloud is used in a question. """ - with self.service_patcher: - service = MockService(backend=BACKEND) + service = MockService(backend=BACKEND) with self.assertRaises(exceptions.FileLocationError): service.ask( @@ -536,46 +512,43 @@ def run_function(*args, **kwargs): with open(temporary_local_path) as f: return MockAnalysis(output_values=f.read()) - with self.service_patcher: - child = MockService(backend=BACKEND, run_function=run_function) - parent = MockService(backend=BACKEND, children={child.id: child}) - child.serve() + child = MockService(backend=BACKEND, run_function=run_function) + parent = MockService(backend=BACKEND, children={child.id: child}) + child.serve() - subscription, _ = parent.ask( - service_id=child.id, - input_values={}, - input_manifest=manifest, - allow_local_files=True, - ) + subscription, _ = parent.ask( + service_id=child.id, + input_values={}, + input_manifest=manifest, + allow_local_files=True, + ) - answer = parent.wait_for_answer(subscription) + answer = parent.wait_for_answer(subscription) self.assertEqual(answer["output_values"], "This is a local file.") def test_ask_with_output_manifest(self): """Test that a service can receive an output manifest as part of the answer to a question.""" - with self.service_patcher: - child = self.make_new_child(BACKEND, run_function_returnee=MockAnalysisWithOutputManifest()) - parent = MockService(backend=BACKEND, children={child.id: child}) - child.serve() + child = self.make_new_child(BACKEND, run_function_returnee=MockAnalysisWithOutputManifest()) + parent = MockService(backend=BACKEND, children={child.id: child}) + child.serve() - subscription, _ = parent.ask(service_id=child.id, input_values={}) - answer = parent.wait_for_answer(subscription) + subscription, _ = parent.ask(service_id=child.id, input_values={}) + answer = parent.wait_for_answer(subscription) self.assertEqual(answer["output_values"], MockAnalysisWithOutputManifest.output_values) self.assertEqual(answer["output_manifest"].id, MockAnalysisWithOutputManifest.output_manifest.id) def test_service_can_ask_multiple_questions_to_child(self): """Test that a service can ask multiple questions to the same child and expect replies to them all.""" - with self.service_patcher: - child = self.make_new_child(BACKEND, run_function_returnee=MockAnalysis()) - parent = MockService(backend=BACKEND, children={child.id: child}) - child.serve() - answers = [] + child = self.make_new_child(BACKEND, run_function_returnee=MockAnalysis()) + parent = MockService(backend=BACKEND, children={child.id: child}) + child.serve() + answers = [] - for i in range(5): - subscription, _ = parent.ask(service_id=child.id, input_values={}) - answers.append(parent.wait_for_answer(subscription, timeout=3600)) + for i in range(5): + subscription, _ = parent.ask(service_id=child.id, input_values={}) + answers.append(parent.wait_for_answer(subscription, timeout=3600)) for answer in answers: self.assertEqual( @@ -585,19 +558,18 @@ def test_service_can_ask_multiple_questions_to_child(self): def test_service_can_ask_questions_to_multiple_children(self): """Test that a service can ask questions to different children and expect replies to them all.""" - with self.service_patcher: - child_1 = self.make_new_child(BACKEND, run_function_returnee=MockAnalysis()) - child_2 = self.make_new_child(BACKEND, run_function_returnee=DifferentMockAnalysis()) - parent = MockService(backend=BACKEND, children={child_1.id: child_1, child_2.id: child_2}) + child_1 = self.make_new_child(BACKEND, run_function_returnee=MockAnalysis()) + child_2 = self.make_new_child(BACKEND, run_function_returnee=DifferentMockAnalysis()) + parent = MockService(backend=BACKEND, children={child_1.id: child_1, child_2.id: child_2}) - child_1.serve() - child_2.serve() + child_1.serve() + child_2.serve() - subscription, _ = parent.ask(service_id=child_1.id, input_values={}) - answer_1 = parent.wait_for_answer(subscription) + subscription, _ = parent.ask(service_id=child_1.id, input_values={}) + answer_1 = parent.wait_for_answer(subscription) - subscription, _ = parent.ask(service_id=child_2.id, input_values={}) - answer_2 = parent.wait_for_answer(subscription) + subscription, _ = parent.ask(service_id=child_2.id, input_values={}) + answer_2 = parent.wait_for_answer(subscription) self.assertEqual( answer_1, @@ -619,26 +591,25 @@ def child_run_function(analysis_id, input_values, *args, **kwargs): subscription, _ = child.ask(service_id=child_of_child.id, input_values=input_values) return MockAnalysis(output_values={input_values["question"]: child.wait_for_answer(subscription)}) - with self.service_patcher: - child_of_child = self.make_new_child(BACKEND, run_function_returnee=DifferentMockAnalysis()) + child_of_child = self.make_new_child(BACKEND, run_function_returnee=DifferentMockAnalysis()) - child = MockService( - backend=BACKEND, - run_function=child_run_function, - children={child_of_child.id: child_of_child}, - ) + child = MockService( + backend=BACKEND, + run_function=child_run_function, + children={child_of_child.id: child_of_child}, + ) - parent = MockService(backend=BACKEND, children={child.id: child}) + parent = MockService(backend=BACKEND, children={child.id: child}) - child.serve() - child_of_child.serve() + child.serve() + child_of_child.serve() - subscription, _ = parent.ask( - service_id=child.id, - input_values={"question": "What does the child of the child say?"}, - ) + subscription, _ = parent.ask( + service_id=child.id, + input_values={"question": "What does the child of the child say?"}, + ) - answer = parent.wait_for_answer(subscription) + answer = parent.wait_for_answer(subscription) self.assertEqual( answer, @@ -667,31 +638,30 @@ def child_run_function(analysis_id, input_values, *args, **kwargs): } ) - with self.service_patcher: - first_child_of_child = self.make_new_child(BACKEND, run_function_returnee=DifferentMockAnalysis()) - second_child_of_child = self.make_new_child(BACKEND, run_function_returnee=MockAnalysis()) + first_child_of_child = self.make_new_child(BACKEND, run_function_returnee=DifferentMockAnalysis()) + second_child_of_child = self.make_new_child(BACKEND, run_function_returnee=MockAnalysis()) - child = MockService( - backend=BACKEND, - run_function=child_run_function, - children={ - first_child_of_child.id: first_child_of_child, - second_child_of_child.id: second_child_of_child, - }, - ) + child = MockService( + backend=BACKEND, + run_function=child_run_function, + children={ + first_child_of_child.id: first_child_of_child, + second_child_of_child.id: second_child_of_child, + }, + ) - parent = MockService(backend=BACKEND, children={child.id: child}) + parent = MockService(backend=BACKEND, children={child.id: child}) - child.serve() - first_child_of_child.serve() - second_child_of_child.serve() + child.serve() + first_child_of_child.serve() + second_child_of_child.serve() - subscription, _ = parent.ask( - service_id=child.id, - input_values={"question": "What does the child of the child say?"}, - ) + subscription, _ = parent.ask( + service_id=child.id, + input_values={"question": "What does the child of the child say?"}, + ) - answer = parent.wait_for_answer(subscription) + answer = parent.wait_for_answer(subscription) self.assertEqual( answer, @@ -712,13 +682,12 @@ def child_run_function(analysis_id, input_values, *args, **kwargs): def test_child_messages_can_be_recorded_by_parent(self): """Test that the parent can record messages it receives from its child to a JSON file.""" - with self.service_patcher: - child = MockService(backend=BACKEND, run_function=self.create_run_function()) - parent = MockService(backend=BACKEND, children={child.id: child}) - child.serve() + child = MockService(backend=BACKEND, run_function=self.create_run_function()) + parent = MockService(backend=BACKEND, children={child.id: child}) + child.serve() - subscription, _ = parent.ask(service_id=child.id, input_values={}, subscribe_to_logs=True) - parent.wait_for_answer(subscription) + subscription, _ = parent.ask(service_id=child.id, input_values={}, subscribe_to_logs=True) + parent.wait_for_answer(subscription) # Check that the child's messages have been recorded by the parent. self.assertEqual(parent.received_messages[0]["kind"], "delivery_acknowledgement") @@ -729,14 +698,13 @@ def test_child_messages_can_be_recorded_by_parent(self): def test_child_exception_message_can_be_recorded_by_parent(self): """Test that the parent can record exceptions raised by the child.""" - with self.service_patcher: - child = self.make_new_child_with_error(ValueError("Oh no.")) - parent = MockService(backend=BACKEND, children={child.id: child}) - child.serve() + child = self.make_new_child_with_error(ValueError("Oh no.")) + parent = MockService(backend=BACKEND, children={child.id: child}) + child.serve() - with self.assertRaises(ValueError): - subscription, _ = parent.ask(service_id=child.id, input_values={}, subscribe_to_logs=True) - parent.wait_for_answer(subscription) + with self.assertRaises(ValueError): + subscription, _ = parent.ask(service_id=child.id, input_values={}, subscribe_to_logs=True) + parent.wait_for_answer(subscription) # Check that the child's messages have been recorded by the parent. self.assertEqual(parent.received_messages[0]["kind"], "delivery_acknowledgement") @@ -751,23 +719,22 @@ def run_function(*args, **kwargs): time.sleep(0.3) return MockAnalysis() - with self.service_patcher: - child = MockService(backend=BACKEND, run_function=lambda *args, **kwargs: run_function()) - parent = MockService(backend=BACKEND, children={child.id: child}) - child.serve() + child = MockService(backend=BACKEND, run_function=lambda *args, **kwargs: run_function()) + parent = MockService(backend=BACKEND, children={child.id: child}) + child.serve() - with patch( - "octue.cloud.emulators._pub_sub.MockService.answer", - functools.partial(child.answer, heartbeat_interval=expected_interval), - ): - subscription, _ = parent.ask( - service_id=child.id, - input_values={}, - subscribe_to_logs=True, - save_diagnostics="SAVE_DIAGNOSTICS_ON_CRASH", - ) + with patch( + "octue.cloud.emulators._pub_sub.MockService.answer", + functools.partial(child.answer, heartbeat_interval=expected_interval), + ): + subscription, _ = parent.ask( + service_id=child.id, + input_values={}, + subscribe_to_logs=True, + save_diagnostics="SAVE_DIAGNOSTICS_ON_CRASH", + ) - parent.wait_for_answer(subscription) + parent.wait_for_answer(subscription) self.assertEqual(parent.received_messages[1]["kind"], "heartbeat") self.assertEqual(parent.received_messages[2]["kind"], "heartbeat") @@ -797,20 +764,19 @@ def run_function(*args, **kwargs): analysis.output_values = {"tada": True} return analysis - with self.service_patcher: - child = MockService(backend=BACKEND, run_function=run_function) - parent = MockService(backend=BACKEND, children={child.id: child}) - child.serve() + child = MockService(backend=BACKEND, run_function=run_function) + parent = MockService(backend=BACKEND, children={child.id: child}) + child.serve() - subscription, _ = parent.ask( - service_id=child.id, - input_values={}, - subscribe_to_logs=True, - save_diagnostics="SAVE_DIAGNOSTICS_ON_CRASH", - ) + subscription, _ = parent.ask( + service_id=child.id, + input_values={}, + subscribe_to_logs=True, + save_diagnostics="SAVE_DIAGNOSTICS_ON_CRASH", + ) - monitor_messages = [] - result = parent.wait_for_answer(subscription, handle_monitor_message=monitor_messages.append) + monitor_messages = [] + result = parent.wait_for_answer(subscription, handle_monitor_message=monitor_messages.append) # Check that multiple monitor messages were sent and received. self.assertTrue(len(monitor_messages) > 1) @@ -850,49 +816,48 @@ def mock_child_app(analysis): service_id=f"octue/child:{MOCK_SERVICE_REVISION_TAG}", ) - with self.service_patcher: - static_child_of_child = self.make_new_child( - backend=BACKEND, - service_id=f"octue/static-child-of-child:{MOCK_SERVICE_REVISION_TAG}", - run_function_returnee=MockAnalysis(output_values="I am the static child."), - ) + static_child_of_child = self.make_new_child( + backend=BACKEND, + service_id=f"octue/static-child-of-child:{MOCK_SERVICE_REVISION_TAG}", + run_function_returnee=MockAnalysis(output_values="I am the static child."), + ) - dynamic_child_of_child = self.make_new_child( - backend=BACKEND, - service_id=f"octue/dynamic-child-of-child:{MOCK_SERVICE_REVISION_TAG}", - run_function_returnee=MockAnalysis(output_values="I am the dynamic child."), - ) + dynamic_child_of_child = self.make_new_child( + backend=BACKEND, + service_id=f"octue/dynamic-child-of-child:{MOCK_SERVICE_REVISION_TAG}", + run_function_returnee=MockAnalysis(output_values="I am the dynamic child."), + ) - child = MockService( - backend=BACKEND, - service_id=f"octue/child:{MOCK_SERVICE_REVISION_TAG}", - run_function=runner.run, - children={ - static_child_of_child.id: static_child_of_child, - dynamic_child_of_child.id: dynamic_child_of_child, - }, - ) + child = MockService( + backend=BACKEND, + service_id=f"octue/child:{MOCK_SERVICE_REVISION_TAG}", + run_function=runner.run, + children={ + static_child_of_child.id: static_child_of_child, + dynamic_child_of_child.id: dynamic_child_of_child, + }, + ) - parent = MockService( - backend=BACKEND, - service_id=f"octue/parent:{MOCK_SERVICE_REVISION_TAG}", - children={child.id: child}, - ) + parent = MockService( + backend=BACKEND, + service_id=f"octue/parent:{MOCK_SERVICE_REVISION_TAG}", + children={child.id: child}, + ) - static_child_of_child.serve() - dynamic_child_of_child.serve() - child.serve() + static_child_of_child.serve() + dynamic_child_of_child.serve() + child.serve() - dynamic_children = [ - { - "key": "expected_child", - "id": f"octue/dynamic-child-of-child:{MOCK_SERVICE_REVISION_TAG}", - "backend": {"name": "GCPPubSubBackend", "project_name": "my-project"}, - }, - ] + dynamic_children = [ + { + "key": "expected_child", + "id": f"octue/dynamic-child-of-child:{MOCK_SERVICE_REVISION_TAG}", + "backend": {"name": "GCPPubSubBackend", "project_name": "my-project"}, + }, + ] - subscription, _ = parent.ask(service_id=child.id, input_values={}, children=dynamic_children) - answer = parent.wait_for_answer(subscription) + subscription, _ = parent.ask(service_id=child.id, input_values={}, children=dynamic_children) + answer = parent.wait_for_answer(subscription) self.assertEqual(answer["output_values"], "I am the dynamic child.") From 38d687f54e82693038c1653bbd9d33d745a5aac3 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 9 Apr 2024 12:45:40 +0100 Subject: [PATCH 117/169] TST: Factor out service patching in pub/sub event handler tests skipci --- tests/cloud/pub_sub/test_events.py | 307 ++++++++++++++--------------- 1 file changed, 153 insertions(+), 154 deletions(-) diff --git a/tests/cloud/pub_sub/test_events.py b/tests/cloud/pub_sub/test_events.py index 4a5c231b0..58806a70d 100644 --- a/tests/cloud/pub_sub/test_events.py +++ b/tests/cloud/pub_sub/test_events.py @@ -12,8 +12,11 @@ class TestPubSubEventHandler(BaseTestCase): + service_patcher = ServicePatcher() + @classmethod def setUpClass(cls): + cls.service_patcher.start() cls.question_uuid = str(uuid.uuid4()) cls.topic = MockTopic(name="octue.services", project_name=TEST_PROJECT_NAME) @@ -25,36 +28,41 @@ def setUpClass(cls): ) cls.subscription.create() - with ServicePatcher(): - cls.parent = MockService( - service_id="my-org/my-service:1.0.0", - backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME), - ) + cls.parent = MockService( + service_id="my-org/my-service:1.0.0", + backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME), + ) + + @classmethod + def tearDownClass(cls): + """Stop the services patcher. + + :return None: + """ + cls.service_patcher.stop() def test_timeout(self): """Test that a TimeoutError is raised if message handling takes longer than the given timeout.""" - with ServicePatcher(): - event_handler = GoogleCloudPubSubEventHandler( - subscription=self.subscription, - receiving_service=self.parent, - event_handlers={"test": lambda message: None, "finish-test": lambda message: message}, - schema={}, - ) + event_handler = GoogleCloudPubSubEventHandler( + subscription=self.subscription, + receiving_service=self.parent, + event_handlers={"test": lambda message: None, "finish-test": lambda message: message}, + schema={}, + ) with self.assertRaises(TimeoutError): event_handler.handle_events(timeout=0) def test_in_order_messages_are_handled_in_order(self): """Test that messages received in order are handled in order.""" - with ServicePatcher(): - event_handler = GoogleCloudPubSubEventHandler( - subscription=self.subscription, - receiving_service=self.parent, - event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, - schema={}, - ) + event_handler = GoogleCloudPubSubEventHandler( + subscription=self.subscription, + receiving_service=self.parent, + event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, + schema={}, + ) - child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) + child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) messages = [ {"event": {"kind": "test"}, "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}}, @@ -84,15 +92,14 @@ def test_in_order_messages_are_handled_in_order(self): def test_out_of_order_messages_are_handled_in_order(self): """Test that messages received out of order are handled in order.""" - with ServicePatcher(): - event_handler = GoogleCloudPubSubEventHandler( - subscription=self.subscription, - receiving_service=self.parent, - event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, - schema={}, - ) + event_handler = GoogleCloudPubSubEventHandler( + subscription=self.subscription, + receiving_service=self.parent, + event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, + schema={}, + ) - child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) + child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) messages = [ { @@ -140,15 +147,14 @@ def test_out_of_order_messages_with_end_message_first_are_handled_in_order(self) """Test that messages received out of order and with the final message (the message that triggers a value to be returned) are handled in order. """ - with ServicePatcher(): - event_handler = GoogleCloudPubSubEventHandler( - subscription=self.subscription, - receiving_service=self.parent, - event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, - schema={}, - ) + event_handler = GoogleCloudPubSubEventHandler( + subscription=self.subscription, + receiving_service=self.parent, + event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, + schema={}, + ) - child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) + child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) messages = [ { @@ -194,15 +200,14 @@ def test_out_of_order_messages_with_end_message_first_are_handled_in_order(self) def test_no_timeout(self): """Test that message handling works with no timeout.""" - with ServicePatcher(): - event_handler = GoogleCloudPubSubEventHandler( - subscription=self.subscription, - receiving_service=self.parent, - event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, - schema={}, - ) + event_handler = GoogleCloudPubSubEventHandler( + subscription=self.subscription, + receiving_service=self.parent, + event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, + schema={}, + ) - child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) + child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) messages = [ { @@ -237,9 +242,8 @@ def test_no_timeout(self): def test_delivery_acknowledgement(self): """Test that a delivery acknowledgement message is handled correctly.""" - with ServicePatcher(): - event_handler = GoogleCloudPubSubEventHandler(subscription=self.subscription, receiving_service=self.parent) - child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) + event_handler = GoogleCloudPubSubEventHandler(subscription=self.subscription, receiving_service=self.parent) + child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) messages = [ { @@ -265,8 +269,7 @@ def test_delivery_acknowledgement(self): def test_error_raised_if_heartbeat_not_received_before_checked(self): """Test that an error is raised if a heartbeat isn't received before a heartbeat is first checked for.""" - with ServicePatcher(): - event_handler = GoogleCloudPubSubEventHandler(subscription=self.subscription, receiving_service=self.parent) + event_handler = GoogleCloudPubSubEventHandler(subscription=self.subscription, receiving_service=self.parent) with self.assertRaises(TimeoutError) as error: event_handler.handle_events(maximum_heartbeat_interval=0) @@ -276,9 +279,7 @@ def test_error_raised_if_heartbeat_not_received_before_checked(self): def test_error_raised_if_heartbeats_stop_being_received(self): """Test that an error is raised if heartbeats stop being received within the maximum interval.""" - with ServicePatcher(): - event_handler = GoogleCloudPubSubEventHandler(subscription=self.subscription, receiving_service=self.parent) - + event_handler = GoogleCloudPubSubEventHandler(subscription=self.subscription, receiving_service=self.parent) event_handler._last_heartbeat = datetime.datetime.now() - datetime.timedelta(seconds=30) with self.assertRaises(TimeoutError) as error: @@ -288,10 +289,8 @@ def test_error_raised_if_heartbeats_stop_being_received(self): def test_error_not_raised_if_heartbeat_has_been_received_in_maximum_allowed_interval(self): """Test that an error is not raised if a heartbeat has been received in the maximum allowed interval.""" - with ServicePatcher(): - event_handler = GoogleCloudPubSubEventHandler(subscription=self.subscription, receiving_service=self.parent) - child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) - + event_handler = GoogleCloudPubSubEventHandler(subscription=self.subscription, receiving_service=self.parent) + child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) event_handler._last_heartbeat = datetime.datetime.now() messages = [ @@ -324,9 +323,7 @@ def test_error_not_raised_if_heartbeat_has_been_received_in_maximum_allowed_inte def test_time_since_last_heartbeat_is_none_if_no_heartbeat_received_yet(self): """Test that the time since the last heartbeat is `None` if no heartbeat has been received yet.""" - with ServicePatcher(): - event_handler = GoogleCloudPubSubEventHandler(subscription=self.subscription, receiving_service=self.parent) - + event_handler = GoogleCloudPubSubEventHandler(subscription=self.subscription, receiving_service=self.parent) self.assertIsNone(event_handler._time_since_last_heartbeat) def test_total_run_time_is_none_if_handle_events_has_not_been_called(self): @@ -345,18 +342,16 @@ def test_missing_messages_at_start_can_be_skipped(self): """Test that missing messages at the start of the event stream can be skipped if they aren't received after a given time period if subsequent messages have been received. """ - with ServicePatcher(): - event_handler = GoogleCloudPubSubEventHandler( - subscription=self.subscription, - receiving_service=self.parent, - event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, - schema={}, - skip_missing_events_after=0, - ) - - child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) + event_handler = GoogleCloudPubSubEventHandler( + subscription=self.subscription, + receiving_service=self.parent, + event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, + schema={}, + skip_missing_events_after=0, + ) # Simulate the first two messages not being received. + child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) child._events_emitted = 2 messages = [ @@ -401,16 +396,15 @@ def test_missing_messages_at_start_can_be_skipped(self): def test_missing_messages_in_middle_can_skipped(self): """Test that missing messages in the middle of the event stream can be skipped.""" - with ServicePatcher(): - event_handler = GoogleCloudPubSubEventHandler( - subscription=self.subscription, - receiving_service=self.parent, - event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, - schema={}, - skip_missing_events_after=0, - ) + event_handler = GoogleCloudPubSubEventHandler( + subscription=self.subscription, + receiving_service=self.parent, + event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, + schema={}, + skip_missing_events_after=0, + ) - child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) + child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) # Send three consecutive messages. messages = [ @@ -462,16 +456,15 @@ def test_missing_messages_in_middle_can_skipped(self): def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): """Test that multiple blocks of missing messages in the middle of the event stream can be skipped.""" - with ServicePatcher(): - event_handler = GoogleCloudPubSubEventHandler( - subscription=self.subscription, - receiving_service=self.parent, - event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, - schema={}, - skip_missing_events_after=0, - ) + event_handler = GoogleCloudPubSubEventHandler( + subscription=self.subscription, + receiving_service=self.parent, + event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, + schema={}, + skip_missing_events_after=0, + ) - child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) + child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) # Send three consecutive messages. messages = [ @@ -558,18 +551,16 @@ def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): def test_all_messages_missing_apart_from_result(self): """Test that the result message is still handled if all other messages are missing.""" - with ServicePatcher(): - event_handler = GoogleCloudPubSubEventHandler( - subscription=self.subscription, - receiving_service=self.parent, - event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, - schema={}, - skip_missing_events_after=0, - ) - - child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) + event_handler = GoogleCloudPubSubEventHandler( + subscription=self.subscription, + receiving_service=self.parent, + event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, + schema={}, + skip_missing_events_after=0, + ) # Simulate missing messages. + child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) child._events_emitted = 1000 # Send the result message. @@ -587,8 +578,11 @@ def test_all_messages_missing_apart_from_result(self): class TestPullAndEnqueueAvailableMessages(BaseTestCase): + service_patcher = ServicePatcher() + @classmethod def setUpClass(cls): + cls.service_patcher.start() cls.question_uuid = str(uuid.uuid4()) cls.topic = MockTopic(name="octue.services", project_name=TEST_PROJECT_NAME) @@ -600,73 +594,78 @@ def setUpClass(cls): ) cls.subscription.create() - with ServicePatcher(): - cls.parent = MockService( - service_id="my-org/my-service:1.0.0", - backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME), - ) + cls.parent = MockService( + service_id="my-org/my-service:1.0.0", + backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME), + ) + + @classmethod + def tearDownClass(cls): + """Stop the services patcher. + + :return None: + """ + cls.service_patcher.stop() def test_pull_and_enqueue_available_events(self): """Test that pulling and enqueuing a message works.""" - with ServicePatcher(): - self.subscription = MockSubscription( - name=f"my-org.my-service.1-0-0.answers.{self.question_uuid}", - topic=self.topic, - ) + self.subscription = MockSubscription( + name=f"my-org.my-service.1-0-0.answers.{self.question_uuid}", + topic=self.topic, + ) - event_handler = GoogleCloudPubSubEventHandler( - subscription=self.subscription, - receiving_service=self.parent, - event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, - schema={}, + event_handler = GoogleCloudPubSubEventHandler( + subscription=self.subscription, + receiving_service=self.parent, + event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, + schema={}, + ) + + event_handler.question_uuid = self.question_uuid + event_handler.child_sruid = "my-org/my-service:1.0.0" + event_handler._child_sdk_version = "0.1.3" + event_handler.waiting_events = {} + + # Enqueue a mock message for a mock subscription to receive. + mock_message = {"kind": "test"} + + MESSAGES[self.question_uuid] = [ + MockMessage.from_primitive( + mock_message, + attributes={ + "order": 0, + "question_uuid": self.question_uuid, + "originator": self.parent.id, + "sender": self.parent.id, + "sender_type": "CHILD", + "sender_sdk_version": "0.50.0", + "recipient": "my-org/my-service:1.0.0", + }, ) + ] - event_handler.question_uuid = self.question_uuid - event_handler.child_sruid = "my-org/my-service:1.0.0" - event_handler._child_sdk_version = "0.1.3" - event_handler.waiting_events = {} - - # Enqueue a mock message for a mock subscription to receive. - mock_message = {"kind": "test"} - - MESSAGES[self.question_uuid] = [ - MockMessage.from_primitive( - mock_message, - attributes={ - "order": 0, - "question_uuid": self.question_uuid, - "originator": self.parent.id, - "sender": self.parent.id, - "sender_type": "CHILD", - "sender_sdk_version": "0.50.0", - "recipient": "my-org/my-service:1.0.0", - }, - ) - ] - - event_handler._pull_and_enqueue_available_events(timeout=10) - self.assertEqual(event_handler.waiting_events, {0: mock_message}) - self.assertEqual(event_handler._earliest_waiting_event_number, 0) + event_handler._pull_and_enqueue_available_events(timeout=10) + self.assertEqual(event_handler.waiting_events, {0: mock_message}) + self.assertEqual(event_handler._earliest_waiting_event_number, 0) def test_timeout_error_raised_if_result_message_not_received_in_time(self): """Test that a timeout error is raised if a result message is not received in time.""" - with ServicePatcher(): - self.subscription = MockSubscription( - name=f"my-org.my-service.1-0-0.answers.{self.question_uuid}", - topic=self.topic, - ) + self.subscription = MockSubscription( + name=f"my-org.my-service.1-0-0.answers.{self.question_uuid}", + topic=self.topic, + ) - event_handler = GoogleCloudPubSubEventHandler( - subscription=self.subscription, - receiving_service=self.parent, - event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, - ) + event_handler = GoogleCloudPubSubEventHandler( + subscription=self.subscription, + receiving_service=self.parent, + event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, + ) - event_handler._child_sdk_version = "0.1.3" - event_handler.waiting_events = {} - event_handler._start_time = 0 + event_handler._child_sdk_version = "0.1.3" + event_handler.waiting_events = {} + event_handler._start_time = 0 - with self.assertRaises(TimeoutError): - event_handler._pull_and_enqueue_available_events(timeout=1e-6) + with self.assertRaises(TimeoutError): + event_handler._pull_and_enqueue_available_events(timeout=1e-6) - self.assertEqual(event_handler._earliest_waiting_event_number, math.inf) + self.assertEqual(event_handler._earliest_waiting_event_number, math.inf) From 19632f3c611fe898c81bbb25ebd0a850cf4e134d Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 9 Apr 2024 14:20:53 +0100 Subject: [PATCH 118/169] FIX: Allow services to be instantiated without GCP credentials --- octue/cloud/pub_sub/service.py | 35 +++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index 346da3148..f2a7821d3 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -85,13 +85,12 @@ def __init__(self, backend, service_id=None, run_function=None, name=None, servi self.backend = backend self.run_function = run_function self.name = name - - self.services_topic = self._get_services_topic() self.service_registries = service_registries self._pub_sub_id = convert_service_id_to_pub_sub_form(self.id) self._local_sdk_version = importlib.metadata.version("octue") self._publisher = None + self._services_topic = None self._event_handler = None self._events_emitted = 0 @@ -115,6 +114,25 @@ def publisher(self): return self._publisher + @property + def services_topic(self): + """Get the Octue services topic that all events in the project are published to. No topic is instantiated until + this property is called for the first time. This allows checking for the `GOOGLE_APPLICATION_CREDENTIALS` + environment variable to be put off until it's needed. + + :raise octue.exceptions.ServiceNotFound: if the topic doesn't exist in the project + :return octue.cloud.pub_sub.topic.Topic: the Octue services topic for the project + """ + if not self._services_topic: + topic = Topic(name=self.backend.services_namespace, project_name=self.backend.project_name) + + if not topic.exists(): + raise octue.exceptions.ServiceNotFound(f"Topic {self.backend.services_namespace!r} cannot be found.") + + self._services_topic = topic + + return self._services_topic + @property def received_messages(self): """Get the messages received by the service from a child service while running the `wait_for_answer` method. If @@ -424,19 +442,6 @@ def send_exception(self, question_uuid, originator, timeout=30): timeout=timeout, ) - def _get_services_topic(self): - """Get the Octue services topic that all events in the project are published to. - - :raise octue.exceptions.ServiceNotFound: if the topic doesn't exist in the project - :return octue.cloud.pub_sub.topic.Topic: the Octue services topic for the project - """ - topic = Topic(name=self.backend.services_namespace, project_name=self.backend.project_name) - - if not topic.exists(): - raise octue.exceptions.ServiceNotFound(f"Topic {self.backend.services_namespace!r} cannot be found.") - - return topic - def _send_message(self, message, originator, recipient, attributes=None, timeout=30): """Send a JSON-serialised event as a Pub/Sub message to the services topic with optional message attributes, incrementing the `_events_emitted` attribute by one. This method is thread-safe. From 0fc802b3a0a062a0ad1d8a888c8069092fce70b5 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 9 Apr 2024 14:22:51 +0100 Subject: [PATCH 119/169] TST: Factor out service patching in child tests --- tests/resources/test_child.py | 218 ++++++++++++++++++---------------- 1 file changed, 113 insertions(+), 105 deletions(-) diff --git a/tests/resources/test_child.py b/tests/resources/test_child.py index 1e9c98234..b80877608 100644 --- a/tests/resources/test_child.py +++ b/tests/resources/test_child.py @@ -8,11 +8,11 @@ from google.auth.exceptions import DefaultCredentialsError -from octue.cloud.emulators._pub_sub import MockAnalysis, MockService, MockSubscriber, MockSubscription, MockTopic +from octue.cloud.emulators._pub_sub import MockAnalysis, MockService, MockTopic from octue.cloud.emulators.child import ServicePatcher from octue.resources.child import Child from octue.resources.service_backends import GCPPubSubBackend -from tests import MOCK_SERVICE_REVISION_TAG +from tests import MOCK_SERVICE_REVISION_TAG, TEST_PROJECT_NAME from tests.base import BaseTestCase @@ -32,6 +32,26 @@ def mock_run_function_that_fails_every_other_time(analysis_id, input_values, *ar class TestChild(BaseTestCase): + service_patcher = ServicePatcher() + + @classmethod + def setUpClass(cls): + """Start the service patcher and create a mock services topic. + + :return None: + """ + cls.service_patcher.start() + topic = MockTopic(name="octue.services", project_name=TEST_PROJECT_NAME) + topic.create(allow_existing=True) + + @classmethod + def tearDownClass(cls): + """Stop the services patcher. + + :return None: + """ + cls.service_patcher.stop() + def test_representation(self): """Test that children are represented correctly as a string.""" self.assertEqual( @@ -55,18 +75,14 @@ def test_instantiating_child_without_credentials(self): def test_child_cannot_be_asked_question_without_credentials(self): """Test that a child cannot be asked a question without Google Cloud credentials being available.""" with patch.dict(os.environ, clear=True): - with patch("octue.cloud.pub_sub.service.Topic", new=MockTopic): - with patch("octue.cloud.pub_sub.service.Subscription", new=MockSubscription): - with patch("octue.resources.child.BACKEND_TO_SERVICE_MAPPING", {"GCPPubSubBackend": MockService}): - with patch("google.cloud.pubsub_v1.SubscriberClient", new=MockSubscriber): - - child = Child( - id=f"octue/my-child:{MOCK_SERVICE_REVISION_TAG}", - backend={"name": "GCPPubSubBackend", "project_name": "blah"}, - ) + with patch("octue.resources.child.BACKEND_TO_SERVICE_MAPPING", {"GCPPubSubBackend": MockService}): + child = Child( + id=f"octue/my-child:{MOCK_SERVICE_REVISION_TAG}", + backend={"name": "GCPPubSubBackend", "project_name": "blah"}, + ) - with self.assertRaises(DefaultCredentialsError): - child.ask({"some": "input"}) + with self.assertRaises(DefaultCredentialsError): + child.ask({"some": "input"}) def test_child_can_be_asked_multiple_questions(self): """Test that a child can be asked multiple questions.""" @@ -76,16 +92,14 @@ def mock_run_function(analysis_id, input_values, *args, **kwargs): responding_service = MockService(backend=GCPPubSubBackend(project_name="blah"), run_function=mock_run_function) - with ServicePatcher(): - with patch("octue.resources.child.BACKEND_TO_SERVICE_MAPPING", {"GCPPubSubBackend": MockService}): - responding_service.serve() - - child = Child(id=responding_service.id, backend={"name": "GCPPubSubBackend", "project_name": "blah"}) + with patch("octue.resources.child.BACKEND_TO_SERVICE_MAPPING", {"GCPPubSubBackend": MockService}): + responding_service.serve() + child = Child(id=responding_service.id, backend={"name": "GCPPubSubBackend", "project_name": "blah"}) - # Make sure the child's underlying mock service knows how to access the mock responding service. - child._service.children[responding_service.id] = responding_service - self.assertEqual(child.ask([1, 2, 3, 4])["output_values"], [1, 2, 3, 4]) - self.assertEqual(child.ask([5, 6, 7, 8])["output_values"], [5, 6, 7, 8]) + # Make sure the child's underlying mock service knows how to access the mock responding service. + child._service.children[responding_service.id] = responding_service + self.assertEqual(child.ask([1, 2, 3, 4])["output_values"], [1, 2, 3, 4]) + self.assertEqual(child.ask([5, 6, 7, 8])["output_values"], [5, 6, 7, 8]) def test_ask_multiple(self): """Test that a child can be asked multiple questions in parallel and return the answers in the correct order.""" @@ -96,27 +110,26 @@ def mock_run_function(analysis_id, input_values, *args, **kwargs): responding_service = MockService(backend=GCPPubSubBackend(project_name="blah"), run_function=mock_run_function) - with ServicePatcher(): - with patch("octue.resources.child.BACKEND_TO_SERVICE_MAPPING", {"GCPPubSubBackend": MockService}): - responding_service.serve() + with patch("octue.resources.child.BACKEND_TO_SERVICE_MAPPING", {"GCPPubSubBackend": MockService}): + responding_service.serve() - child = Child(id=responding_service.id, backend={"name": "GCPPubSubBackend", "project_name": "blah"}) + child = Child(id=responding_service.id, backend={"name": "GCPPubSubBackend", "project_name": "blah"}) - # Make sure the child's underlying mock service knows how to access the mock responding service. - child._service.children[responding_service.id] = responding_service + # Make sure the child's underlying mock service knows how to access the mock responding service. + child._service.children[responding_service.id] = responding_service - answers = child.ask_multiple( - {"input_values": [1, 2, 3, 4]}, - {"input_values": [5, 6, 7, 8]}, - ) + answers = child.ask_multiple( + {"input_values": [1, 2, 3, 4]}, + {"input_values": [5, 6, 7, 8]}, + ) - self.assertEqual( - answers, - [ - {"output_values": [1, 2, 3, 4], "output_manifest": None}, - {"output_values": [5, 6, 7, 8], "output_manifest": None}, - ], - ) + self.assertEqual( + answers, + [ + {"output_values": [1, 2, 3, 4], "output_manifest": None}, + {"output_values": [5, 6, 7, 8], "output_manifest": None}, + ], + ) def test_error_raised_in_ask_multiple_if_one_question_fails_when_raise_errors_is_true(self): """Test that an error is raised if any of the questions given to `Child.ask_multiple` fail when `raise_errors` @@ -137,21 +150,20 @@ def mock_run_function_that_sometimes_fails(analysis_id, input_values, *args, **k run_function=functools.partial(mock_run_function_that_sometimes_fails, runs=Value("d", 0)), ) - with ServicePatcher(): - with patch("octue.resources.child.BACKEND_TO_SERVICE_MAPPING", {"GCPPubSubBackend": MockService}): - responding_service.serve() + with patch("octue.resources.child.BACKEND_TO_SERVICE_MAPPING", {"GCPPubSubBackend": MockService}): + responding_service.serve() - child = Child(id=responding_service.id, backend={"name": "GCPPubSubBackend", "project_name": "blah"}) + child = Child(id=responding_service.id, backend={"name": "GCPPubSubBackend", "project_name": "blah"}) - # Make sure the child's underlying mock service knows how to access the mock responding service. - child._service.children[responding_service.id] = responding_service + # Make sure the child's underlying mock service knows how to access the mock responding service. + child._service.children[responding_service.id] = responding_service - with self.assertRaises(ValueError): - child.ask_multiple( - {"input_values": [1, 2, 3, 4]}, - {"input_values": [5, 6, 7, 8]}, - {"input_values": [9, 10, 11, 12]}, - ) + with self.assertRaises(ValueError): + child.ask_multiple( + {"input_values": [1, 2, 3, 4]}, + {"input_values": [5, 6, 7, 8]}, + {"input_values": [9, 10, 11, 12]}, + ) def test_error_not_raised_by_ask_multiple_if_one_question_fails_when_raise_errors_is_false(self): """Test that an error is not raised if one of the questions given to `Child.ask_multiple` fail when @@ -162,21 +174,20 @@ def test_error_not_raised_by_ask_multiple_if_one_question_fails_when_raise_error run_function=functools.partial(mock_run_function_that_fails_every_other_time, runs=Value("d", 0)), ) - with ServicePatcher(): - with patch("octue.resources.child.BACKEND_TO_SERVICE_MAPPING", {"GCPPubSubBackend": MockService}): - responding_service.serve() + with patch("octue.resources.child.BACKEND_TO_SERVICE_MAPPING", {"GCPPubSubBackend": MockService}): + responding_service.serve() - child = Child(id=responding_service.id, backend={"name": "GCPPubSubBackend", "project_name": "blah"}) + child = Child(id=responding_service.id, backend={"name": "GCPPubSubBackend", "project_name": "blah"}) - # Make sure the child's underlying mock service knows how to access the mock responding service. - child._service.children[responding_service.id] = responding_service + # Make sure the child's underlying mock service knows how to access the mock responding service. + child._service.children[responding_service.id] = responding_service - answers = child.ask_multiple( - {"input_values": [1, 2, 3, 4]}, - {"input_values": [5, 6, 7, 8]}, - {"input_values": [9, 10, 11, 12]}, - raise_errors=False, - ) + answers = child.ask_multiple( + {"input_values": [1, 2, 3, 4]}, + {"input_values": [5, 6, 7, 8]}, + {"input_values": [9, 10, 11, 12]}, + raise_errors=False, + ) successful_answers = [] failed_answers = [] @@ -204,22 +215,21 @@ def test_ask_multiple_with_failed_question_retry(self): run_function=functools.partial(mock_run_function_that_fails_every_other_time, runs=Value("d", 0)), ) - with ServicePatcher(): - with patch("octue.resources.child.BACKEND_TO_SERVICE_MAPPING", {"GCPPubSubBackend": MockService}): - responding_service.serve() + with patch("octue.resources.child.BACKEND_TO_SERVICE_MAPPING", {"GCPPubSubBackend": MockService}): + responding_service.serve() - child = Child(id=responding_service.id, backend={"name": "GCPPubSubBackend", "project_name": "blah"}) + child = Child(id=responding_service.id, backend={"name": "GCPPubSubBackend", "project_name": "blah"}) - # Make sure the child's underlying mock service knows how to access the mock responding service. - child._service.children[responding_service.id] = responding_service + # Make sure the child's underlying mock service knows how to access the mock responding service. + child._service.children[responding_service.id] = responding_service - # Only ask two questions so the question success/failure order plays out as desired. - answers = child.ask_multiple( - {"input_values": [1, 2, 3, 4]}, - {"input_values": [5, 6, 7, 8]}, - raise_errors=False, - max_retries=1, - ) + # Only ask two questions so the question success/failure order plays out as desired. + answers = child.ask_multiple( + {"input_values": [1, 2, 3, 4]}, + {"input_values": [5, 6, 7, 8]}, + raise_errors=False, + max_retries=1, + ) # Check that both questions succeeded. self.assertEqual( @@ -247,24 +257,23 @@ def test_ask_multiple_with_multiple_failed_question_retries(self): run_function=functools.partial(mock_run_function_that_fails_every_other_time, runs=Value("d", 0)), ) - with ServicePatcher(): - with patch("octue.resources.child.BACKEND_TO_SERVICE_MAPPING", {"GCPPubSubBackend": MockService}): - responding_service.serve() + with patch("octue.resources.child.BACKEND_TO_SERVICE_MAPPING", {"GCPPubSubBackend": MockService}): + responding_service.serve() - child = Child(id=responding_service.id, backend={"name": "GCPPubSubBackend", "project_name": "blah"}) + child = Child(id=responding_service.id, backend={"name": "GCPPubSubBackend", "project_name": "blah"}) - # Make sure the child's underlying mock service knows how to access the mock responding service. - child._service.children[responding_service.id] = responding_service + # Make sure the child's underlying mock service knows how to access the mock responding service. + child._service.children[responding_service.id] = responding_service - # Only ask two questions so the question success/failure order plays out as desired. - answers = child.ask_multiple( - {"input_values": [1, 2, 3, 4]}, - {"input_values": [5, 6, 7, 8]}, - {"input_values": [9, 10, 11, 12]}, - {"input_values": [13, 14, 15, 16]}, - raise_errors=False, - max_retries=2, - ) + # Only ask two questions so the question success/failure order plays out as desired. + answers = child.ask_multiple( + {"input_values": [1, 2, 3, 4]}, + {"input_values": [5, 6, 7, 8]}, + {"input_values": [9, 10, 11, 12]}, + {"input_values": [13, 14, 15, 16]}, + raise_errors=False, + max_retries=2, + ) # Check that all four questions succeeded. self.assertEqual( @@ -284,23 +293,22 @@ def test_ask_multiple_with_prevented_retries(self): run_function=functools.partial(mock_run_function_that_fails_every_other_time, runs=Value("d", 0)), ) - with ServicePatcher(): - with patch("octue.resources.child.BACKEND_TO_SERVICE_MAPPING", {"GCPPubSubBackend": MockService}): - responding_service.serve() + with patch("octue.resources.child.BACKEND_TO_SERVICE_MAPPING", {"GCPPubSubBackend": MockService}): + responding_service.serve() - child = Child(id=responding_service.id, backend={"name": "GCPPubSubBackend", "project_name": "blah"}) + child = Child(id=responding_service.id, backend={"name": "GCPPubSubBackend", "project_name": "blah"}) - # Make sure the child's underlying mock service knows how to access the mock responding service. - child._service.children[responding_service.id] = responding_service + # Make sure the child's underlying mock service knows how to access the mock responding service. + child._service.children[responding_service.id] = responding_service - # Only ask two questions so the question success/failure order plays out as desired. - answers = child.ask_multiple( - {"input_values": [1, 2, 3, 4]}, - {"input_values": [5, 6, 7, 8]}, - raise_errors=False, - max_retries=1, - prevent_retries_when=[ValueError], - ) + # Only ask two questions so the question success/failure order plays out as desired. + answers = child.ask_multiple( + {"input_values": [1, 2, 3, 4]}, + {"input_values": [5, 6, 7, 8]}, + raise_errors=False, + max_retries=1, + prevent_retries_when=[ValueError], + ) successful_answers = [] failed_answers = [] From aa37402fe1425fa7f4bbbf21646b6db386cb0ed8 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 9 Apr 2024 15:12:14 +0100 Subject: [PATCH 120/169] ENH: Add string representation to `MockMessageWrapper` --- octue/cloud/emulators/_pub_sub.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/octue/cloud/emulators/_pub_sub.py b/octue/cloud/emulators/_pub_sub.py index 88f9816b6..527ceb45b 100644 --- a/octue/cloud/emulators/_pub_sub.py +++ b/octue/cloud/emulators/_pub_sub.py @@ -242,6 +242,13 @@ def __init__(self, message): self.message = message self.ack_id = None + def __repr__(self): + """Represent the mock message as a string. + + :return str: + """ + return f"<{type(self).__name__}(message={self.message})>" + class MockMessage: """A mock Pub/Sub message containing serialised data and any number of attributes. From 426e76554184b677b2a385bbafe94a77235980bd Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 9 Apr 2024 15:48:26 +0100 Subject: [PATCH 121/169] FIX: Fix race condition in event emission order --- octue/cloud/events/counter.py | 35 +++++++++++++++ octue/cloud/pub_sub/logging.py | 5 ++- octue/cloud/pub_sub/service.py | 43 +++++++++++------- tests/cloud/pub_sub/test_events.py | 68 +++++++++++++++++++---------- tests/cloud/pub_sub/test_logging.py | 27 +++++++++--- 5 files changed, 131 insertions(+), 47 deletions(-) create mode 100644 octue/cloud/events/counter.py diff --git a/octue/cloud/events/counter.py b/octue/cloud/events/counter.py new file mode 100644 index 000000000..92cec1d7a --- /dev/null +++ b/octue/cloud/events/counter.py @@ -0,0 +1,35 @@ +class EventCounter: + """A mutable counter for keeping track of the emission order of events. This is used in the `Service` class instead + of an integer because it is mutable and can be passed to the `Service._send_message` method and incremented as + events are emitted. + + :return None: + """ + + def __init__(self): + self.count = 0 + + def __iadd__(self, other): + """Increment the counter by an integer. + + :return octue.cloud.events.counter.EventCounter: the event counter with its count updated + """ + if not isinstance(other, int): + raise ValueError(f"Event counters can only be incremented by an integer; received {other!r}.") + + self.count += other + return self + + def __int__(self): + """Get the counter as an integer. + + :return int: the counter as an integer + """ + return int(self.count) + + def __repr__(self): + """Represent the counter as a string. + + :return str: the counter represented as a string. + """ + return f"<{type(self).__name__}(count={self.count})" diff --git a/octue/cloud/pub_sub/logging.py b/octue/cloud/pub_sub/logging.py index f28906702..80f9a744d 100644 --- a/octue/cloud/pub_sub/logging.py +++ b/octue/cloud/pub_sub/logging.py @@ -12,15 +12,17 @@ class GoogleCloudPubSubHandler(logging.Handler): :param str question_uuid: the UUID of the question to handle log records for :param str originator: the SRUID of the service that asked the question these log records are related to :param str recipient: the SRUID of the service to send these log records to + :param octue.cloud.events.counter.EventCounter order: an event counter keeping track of the order of emitted events :param float timeout: timeout in seconds for attempting to publish each log record :return None: """ - def __init__(self, message_sender, question_uuid, originator, recipient, timeout=60, *args, **kwargs): + def __init__(self, message_sender, question_uuid, originator, recipient, order, timeout=60, *args, **kwargs): super().__init__(*args, **kwargs) self.question_uuid = question_uuid self.originator = originator self.recipient = recipient + self.order = order self.timeout = timeout self._send_message = message_sender @@ -38,6 +40,7 @@ def emit(self, record): }, originator=self.originator, recipient=self.recipient, + order=self.order, attributes={ "question_uuid": self.question_uuid, "sender_type": "CHILD", # The sender type is repeated here as a string to avoid a circular import. diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index f2a7821d3..079fdd67b 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -14,6 +14,7 @@ from google.cloud import pubsub_v1 import octue.exceptions +from octue.cloud.events.counter import EventCounter from octue.cloud.events.validation import raise_if_event_is_invalid from octue.cloud.pub_sub import Subscription, Topic from octue.cloud.pub_sub.events import GoogleCloudPubSubEventHandler, extract_event_and_attributes_from_pub_sub_message @@ -92,7 +93,6 @@ def __init__(self, backend, service_id=None, run_function=None, name=None, servi self._publisher = None self._services_topic = None self._event_handler = None - self._events_emitted = 0 def __repr__(self): """Represent the service as a string. @@ -225,15 +225,15 @@ def answer(self, question, heartbeat_interval=120, timeout=30): return heartbeater = None - self._events_emitted = 0 + order = EventCounter() try: - self._send_delivery_acknowledgment(question_uuid, originator) + self._send_delivery_acknowledgment(question_uuid, originator, order) heartbeater = RepeatingTimer( interval=heartbeat_interval, function=self._send_heartbeat, - kwargs={"question_uuid": question_uuid, "originator": originator}, + kwargs={"question_uuid": question_uuid, "originator": originator, "order": order}, ) heartbeater.daemon = True @@ -245,6 +245,7 @@ def answer(self, question, heartbeat_interval=120, timeout=30): question_uuid=question_uuid, originator=originator, recipient=originator, + order=order, ) else: analysis_log_handler = None @@ -259,6 +260,7 @@ def answer(self, question, heartbeat_interval=120, timeout=30): self._send_monitor_message, question_uuid=question_uuid, originator=originator, + order=order, ), save_diagnostics=save_diagnostics, ) @@ -272,6 +274,7 @@ def answer(self, question, heartbeat_interval=120, timeout=30): message=result, originator=originator, recipient=originator, + order=order, attributes={"question_uuid": question_uuid, "sender_type": CHILD_SENDER_TYPE}, timeout=timeout, ) @@ -284,7 +287,7 @@ def answer(self, question, heartbeat_interval=120, timeout=30): heartbeater.cancel() warn_if_incompatible(child_sdk_version=self._local_sdk_version, parent_sdk_version=parent_sdk_version) - self.send_exception(question_uuid, originator, timeout=timeout) + self.send_exception(question_uuid, originator, order, timeout=timeout) raise error def ask( @@ -362,8 +365,6 @@ def ask( ) answer_subscription.create(allow_existing=False) - self._events_emitted = 0 - self._send_question( input_values=input_values, input_manifest=input_manifest, @@ -418,11 +419,12 @@ def wait_for_answer( finally: subscription.delete() - def send_exception(self, question_uuid, originator, timeout=30): + def send_exception(self, question_uuid, originator, order, timeout=30): """Serialise and send the exception being handled to the parent. :param str question_uuid: :param str originator: the SRUID of the service that asked the question this event is related to + :param octue.cloud.events.counter.EventCounter order: an event counter keeping track of the order of emitted events :param float|None timeout: time in seconds to keep retrying sending of the exception :return None: """ @@ -438,17 +440,19 @@ def send_exception(self, question_uuid, originator, timeout=30): }, originator=originator, recipient=originator, + order=order, attributes={"question_uuid": question_uuid, "sender_type": CHILD_SENDER_TYPE}, timeout=timeout, ) - def _send_message(self, message, originator, recipient, attributes=None, timeout=30): + def _send_message(self, message, originator, recipient, order, attributes=None, timeout=30): """Send a JSON-serialised event as a Pub/Sub message to the services topic with optional message attributes, - incrementing the `_events_emitted` attribute by one. This method is thread-safe. + incrementing the `order` argument by one. This method is thread-safe. :param dict message: JSON-serialisable data to send as a message :param str originator: the SRUID of the service that asked the question this event is related to :param str recipient: the SRUID of the service the event is intended for + :param octue.cloud.events.counter.EventCounter order: an event counter keeping track of the order of emitted events :param dict|None attributes: key-value pairs to attach to the event - the values must be strings or bytes :param int|float timeout: the timeout for sending the event in seconds :return google.cloud.pubsub_v1.publisher.futures.Future: @@ -461,7 +465,7 @@ def _send_message(self, message, originator, recipient, attributes=None, timeout attributes["recipient"] = recipient with emit_event_lock: - attributes["order"] = self._events_emitted + attributes["order"] = int(order) converted_attributes = {} for key, value in attributes.items(): @@ -479,7 +483,7 @@ def _send_message(self, message, originator, recipient, attributes=None, timeout **converted_attributes, ) - self._events_emitted += 1 + order += 1 return future @@ -517,6 +521,7 @@ def _send_question( timeout=timeout, originator=self.id, recipient=service_id, + order=EventCounter(), attributes={ "question_uuid": question_uuid, "forward_logs": forward_logs, @@ -529,11 +534,12 @@ def _send_question( future.result() logger.info("%r asked a question %r to service %r.", self, question_uuid, service_id) - def _send_delivery_acknowledgment(self, question_uuid, originator, timeout=30): + def _send_delivery_acknowledgment(self, question_uuid, originator, order, timeout=30): """Send an acknowledgement of question receipt to the parent. :param str question_uuid: :param str originator: the SRUID of the service that asked the question this event is related to + :param octue.cloud.events.counter.EventCounter order: an event counter keeping track of the order of emitted events :param float timeout: time in seconds after which to give up sending :return None: """ @@ -545,16 +551,18 @@ def _send_delivery_acknowledgment(self, question_uuid, originator, timeout=30): timeout=timeout, originator=originator, recipient=originator, + order=order, attributes={"question_uuid": question_uuid, "sender_type": CHILD_SENDER_TYPE}, ) logger.info("%r acknowledged receipt of question %r.", self, question_uuid) - def _send_heartbeat(self, question_uuid, originator, timeout=30): + def _send_heartbeat(self, question_uuid, originator, order, timeout=30): """Send a heartbeat to the parent, indicating that the service is alive. :param str question_uuid: :param str originator: the SRUID of the service that asked the question this event is related to + :param octue.cloud.events.counter.EventCounter order: an event counter keeping track of the order of emitted events :param float timeout: time in seconds after which to give up sending :return None: """ @@ -565,18 +573,20 @@ def _send_heartbeat(self, question_uuid, originator, timeout=30): }, originator=originator, recipient=originator, + order=order, timeout=timeout, attributes={"question_uuid": question_uuid, "sender_type": CHILD_SENDER_TYPE}, ) logger.debug("Heartbeat sent by %r.", self) - def _send_monitor_message(self, data, question_uuid, originator, timeout=30): + def _send_monitor_message(self, data, question_uuid, originator, order, timeout=30): """Send a monitor message to the parent. :param any data: the data to send as a monitor message :param str question_uuid: :param str originator: the SRUID of the service that asked the question this event is related to + :param octue.cloud.events.counter.EventCounter order: an event counter keeping track of the order of emitted events :param float timeout: time in seconds to retry sending the message :return None: """ @@ -584,6 +594,7 @@ def _send_monitor_message(self, data, question_uuid, originator, timeout=30): {"kind": "monitor_message", "data": data}, originator=originator, recipient=originator, + order=order, timeout=timeout, attributes={"question_uuid": question_uuid, "sender_type": CHILD_SENDER_TYPE}, ) @@ -613,7 +624,7 @@ def _parse_question(self, question): child_sdk_version=importlib.metadata.version("octue"), ) - logger.info("%r parsed the question successfully.", self) + logger.info("%r parsed question %r successfully.", self, attributes["question_uuid"]) return ( event, diff --git a/tests/cloud/pub_sub/test_events.py b/tests/cloud/pub_sub/test_events.py index 58806a70d..1ffb60025 100644 --- a/tests/cloud/pub_sub/test_events.py +++ b/tests/cloud/pub_sub/test_events.py @@ -65,11 +65,20 @@ def test_in_order_messages_are_handled_in_order(self): child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) messages = [ - {"event": {"kind": "test"}, "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}}, - {"event": {"kind": "test"}, "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}}, - {"event": {"kind": "test"}, "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}}, { - "event": {"kind": "finish-test"}, + "event": {"kind": "test", "order": 0}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, + }, + { + "event": {"kind": "test", "order": 1}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, + }, + { + "event": {"kind": "test", "order": 2}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, + }, + { + "event": {"kind": "finish-test", "order": 3}, "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, }, ] @@ -80,6 +89,7 @@ def test_in_order_messages_are_handled_in_order(self): attributes=message["attributes"], originator=self.parent.id, recipient=self.parent.id, + order=message["event"]["order"], ) result = event_handler.handle_events() @@ -87,7 +97,12 @@ def test_in_order_messages_are_handled_in_order(self): self.assertEqual( event_handler.handled_events, - [{"kind": "test"}, {"kind": "test"}, {"kind": "test"}, {"kind": "finish-test"}], + [ + {"kind": "test", "order": 0}, + {"kind": "test", "order": 1}, + {"kind": "test", "order": 2}, + {"kind": "finish-test", "order": 3}, + ], ) def test_out_of_order_messages_are_handled_in_order(self): @@ -121,12 +136,12 @@ def test_out_of_order_messages_are_handled_in_order(self): ] for message in messages: - child._events_emitted = message["event"]["order"] child._send_message( message=message["event"], attributes=message["attributes"], originator=self.parent.id, recipient=self.parent.id, + order=message["event"]["order"], ) result = event_handler.handle_events() @@ -176,12 +191,12 @@ def test_out_of_order_messages_with_end_message_first_are_handled_in_order(self) ] for message in messages: - child._events_emitted = message["event"]["order"] child._send_message( message=message["event"], attributes=message["attributes"], originator=self.parent.id, recipient=self.parent.id, + order=message["event"]["order"], ) result = event_handler.handle_events() @@ -230,6 +245,7 @@ def test_no_timeout(self): attributes=message["attributes"], originator=self.parent.id, recipient=self.parent.id, + order=message["event"]["order"], ) result = event_handler.handle_events(timeout=None) @@ -247,11 +263,15 @@ def test_delivery_acknowledgement(self): messages = [ { - "event": {"kind": "delivery_acknowledgement", "datetime": datetime.datetime.utcnow().isoformat()}, + "event": { + "kind": "delivery_acknowledgement", + "datetime": datetime.datetime.utcnow().isoformat(), + "order": 0, + }, "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, }, { - "event": {"kind": "result"}, + "event": {"kind": "result", "order": 1}, "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, }, ] @@ -262,6 +282,7 @@ def test_delivery_acknowledgement(self): attributes=message["attributes"], originator=self.parent.id, recipient=self.parent.id, + order=message["event"]["order"], ) result = event_handler.handle_events() @@ -298,11 +319,12 @@ def test_error_not_raised_if_heartbeat_has_been_received_in_maximum_allowed_inte "event": { "kind": "delivery_acknowledgement", "datetime": datetime.datetime.utcnow().isoformat(), + "order": 0, }, "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, }, { - "event": {"kind": "result"}, + "event": {"kind": "result", "order": 1}, "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, }, ] @@ -313,6 +335,7 @@ def test_error_not_raised_if_heartbeat_has_been_received_in_maximum_allowed_inte attributes=message["attributes"], originator=self.parent.id, recipient=self.parent.id, + order=message["event"]["order"], ) with patch( @@ -350,10 +373,9 @@ def test_missing_messages_at_start_can_be_skipped(self): skip_missing_events_after=0, ) - # Simulate the first two messages not being received. child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) - child._events_emitted = 2 + # Simulate the first two messages not being received. messages = [ { "event": {"kind": "test", "order": 2}, @@ -379,6 +401,7 @@ def test_missing_messages_at_start_can_be_skipped(self): attributes=message["attributes"], originator=self.parent.id, recipient=self.parent.id, + order=message["event"]["order"], ) result = event_handler.handle_events() @@ -428,17 +451,17 @@ def test_missing_messages_in_middle_can_skipped(self): attributes=message["attributes"], originator=self.parent.id, recipient=self.parent.id, + order=message["event"]["order"], ) - # Simulate missing messages. - child._events_emitted = 5 - # Send a final message. child._send_message( message={"kind": "finish-test", "order": 5}, attributes={"question_uuid": self.question_uuid, "sender_type": "CHILD"}, originator=self.parent.id, recipient=self.parent.id, + # Simulate missing messages. + order=5, ) event_handler.handle_events() @@ -488,22 +511,19 @@ def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): attributes=message["attributes"], originator=self.parent.id, recipient=self.parent.id, + order=message["event"]["order"], ) - # Simulate missing messages. - child._events_emitted = 5 - # Send another message. child._send_message( message={"kind": "test", "order": 5}, attributes={"order": 5, "question_uuid": self.question_uuid, "sender_type": "CHILD"}, originator=self.parent.id, recipient=self.parent.id, + # Simulate missing messages. + order=5, ) - # Simulate more missing messages. - child._events_emitted = 20 - # Send more consecutive messages. messages = [ { @@ -530,6 +550,8 @@ def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): attributes=message["attributes"], originator=self.parent.id, recipient=self.parent.id, + # Simulate more missing messages. + order=message["event"]["order"], ) event_handler.handle_events() @@ -559,9 +581,7 @@ def test_all_messages_missing_apart_from_result(self): skip_missing_events_after=0, ) - # Simulate missing messages. child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) - child._events_emitted = 1000 # Send the result message. child._send_message( @@ -569,6 +589,8 @@ def test_all_messages_missing_apart_from_result(self): attributes={"question_uuid": self.question_uuid, "sender_type": "CHILD"}, originator=self.parent.id, recipient=self.parent.id, + # Simulate missing messages. + order=1000, ) event_handler.handle_events() diff --git a/tests/cloud/pub_sub/test_logging.py b/tests/cloud/pub_sub/test_logging.py index 7b2f0f360..c60b7bca2 100644 --- a/tests/cloud/pub_sub/test_logging.py +++ b/tests/cloud/pub_sub/test_logging.py @@ -5,6 +5,7 @@ from octue.cloud.emulators._pub_sub import MESSAGES, MockService, MockTopic from octue.cloud.emulators.child import ServicePatcher +from octue.cloud.events.counter import EventCounter from octue.cloud.pub_sub.logging import GoogleCloudPubSubHandler from octue.resources.service_backends import GCPPubSubBackend from tests import TEST_PROJECT_NAME @@ -17,26 +18,38 @@ def __repr__(self): class TestGoogleCloudPubSubHandler(BaseTestCase): + service_patcher = ServicePatcher() + @classmethod def setUpClass(cls): + """Start the service patcher and create a mock services topic. + + :return None: + """ + cls.service_patcher.start() topic = MockTopic(name="octue.services", project_name=TEST_PROJECT_NAME) + topic.create(allow_existing=True) - with ServicePatcher(): - topic.create(allow_existing=True) + @classmethod + def tearDownClass(cls): + """Stop the services patcher. + + :return None: + """ + cls.service_patcher.stop() def test_emit(self): """Test the log message is published when `GoogleCloudPubSubHandler.emit` is called.""" question_uuid = "96d69278-44ac-4631-aeea-c90fb08a1b2b" log_record = makeLogRecord({"msg": "Starting analysis."}) - - with ServicePatcher(): - service = MockService(backend=GCPPubSubBackend(project_name="blah")) + service = MockService(backend=GCPPubSubBackend(project_name="blah")) GoogleCloudPubSubHandler( message_sender=service._send_message, question_uuid=question_uuid, originator="another/service:1.0.0", recipient="another/service:1.0.0", + order=EventCounter(), ).emit(log_record) self.assertEqual( @@ -58,8 +71,7 @@ def test_emit_with_non_json_serialisable_args(self): {"msg": "%r is not JSON-serialisable but can go into a log message", "args": (non_json_serialisable_thing,)} ) - with ServicePatcher(): - service = MockService(backend=GCPPubSubBackend(project_name="blah")) + service = MockService(backend=GCPPubSubBackend(project_name="blah")) with patch("octue.cloud.emulators._pub_sub.MockPublisher.publish") as mock_publish: GoogleCloudPubSubHandler( @@ -67,6 +79,7 @@ def test_emit_with_non_json_serialisable_args(self): question_uuid="question-uuid", originator="another/service:1.0.0", recipient="another/service:1.0.0", + order=EventCounter(), ).emit(record) self.assertEqual( From 67a24903922886259e0981b91ea4aee8281ce9c5 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 9 Apr 2024 16:03:13 +0100 Subject: [PATCH 122/169] REF: Rename `receiving_service` to `recipient` --- octue/cloud/events/handler.py | 20 ++++++++--------- octue/cloud/events/replayer.py | 6 ++--- octue/cloud/events/validation.py | 19 +++++----------- octue/cloud/pub_sub/events.py | 6 ++--- octue/cloud/pub_sub/service.py | 4 ++-- tests/cloud/pub_sub/test_events.py | 36 +++++++++++++++--------------- 6 files changed, 42 insertions(+), 49 deletions(-) diff --git a/octue/cloud/events/handler.py b/octue/cloud/events/handler.py index 34648ef1c..62c2d489f 100644 --- a/octue/cloud/events/handler.py +++ b/octue/cloud/events/handler.py @@ -31,7 +31,7 @@ class AbstractEventHandler: """An abstract event handler. Inherit from this and add the `handle_events` and `_extract_event_and_attributes` methods to handle events from a specific source synchronously or asynchronously. - :param octue.cloud.pub_sub.service.Service receiving_service: the service that's receiving the events + :param octue.cloud.pub_sub.service.Service recipient: the service that's receiving the events :param callable|None handle_monitor_message: a function to handle monitor messages (e.g. send them to an endpoint for plotting or displaying) - this function should take a single JSON-compatible python primitive :param bool record_events: if `True`, record received events in the `received_events` attribute :param dict|None event_handlers: a mapping of event type names to callables that handle each type of event. The handlers should not mutate the events. @@ -43,7 +43,7 @@ class AbstractEventHandler: def __init__( self, - receiving_service, + recipient, handle_monitor_message=None, record_events=True, event_handlers=None, @@ -51,7 +51,7 @@ def __init__( skip_missing_events_after=10, only_handle_result=False, ): - self.receiving_service = receiving_service + self.recipient = recipient self.handle_monitor_message = handle_monitor_message self.record_events = record_events self.schema = schema @@ -122,7 +122,7 @@ def _extract_and_enqueue_event(self, container): if not is_event_valid( event=event, attributes=attributes, - receiving_service=self.receiving_service, + recipient=self.recipient, parent_sdk_version=PARENT_SDK_VERSION, child_sdk_version=attributes["sender_sdk_version"], schema=self.schema, @@ -135,13 +135,13 @@ def _extract_and_enqueue_event(self, container): self.question_uuid = attributes["question_uuid"] self._child_sdk_version = attributes["sender_sdk_version"] - logger.debug("%r received an event related to question %r.", self.receiving_service, self.question_uuid) + logger.debug("%r received an event related to question %r.", self.recipient, self.question_uuid) order = attributes["order"] if order in self.waiting_events: logger.warning( "%r: Event with duplicate order %d received for question %s - overwriting original event.", - self.receiving_service, + self.recipient, order, self.question_uuid, ) @@ -202,7 +202,7 @@ def _skip_to_earliest_waiting_event(self): logger.warning( "%r: %d consecutive events missing for question %r after %ds - skipping to next earliest waiting event " "(event %d).", - self.receiving_service, + self.recipient, number_of_missing_events, self.question_uuid, self.skip_missing_events_after, @@ -234,7 +234,7 @@ def _handle_delivery_acknowledgement(self, event): :param dict event: :return None: """ - logger.info("%r's question was delivered at %s.", self.receiving_service, event["datetime"]) + logger.info("%r's question was delivered at %s.", self.recipient, event["datetime"]) def _handle_heartbeat(self, event): """Record the time the heartbeat was received. @@ -251,7 +251,7 @@ def _handle_monitor_message(self, event): :param dict event: :return None: """ - logger.debug("%r received a monitor message.", self.receiving_service) + logger.debug("%r received a monitor message.", self.recipient) if self.handle_monitor_message is not None: self.handle_monitor_message(event["data"]) @@ -315,7 +315,7 @@ def _handle_result(self, event): :param dict event: :return dict: """ - logger.info("%r received an answer to question %r.", self.receiving_service, self.question_uuid) + logger.info("%r received an answer to question %r.", self.recipient, self.question_uuid) if event.get("output_manifest"): output_manifest = Manifest.deserialise(event["output_manifest"]) diff --git a/octue/cloud/events/replayer.py b/octue/cloud/events/replayer.py index 8f83c5da6..fb1dec272 100644 --- a/octue/cloud/events/replayer.py +++ b/octue/cloud/events/replayer.py @@ -12,7 +12,7 @@ class EventReplayer(AbstractEventHandler): """A replayer for events retrieved asynchronously from some kind of storage. - :param octue.cloud.pub_sub.service.Service receiving_service: the service that's receiving the events + :param octue.cloud.pub_sub.service.Service recipient: the service that's receiving the events :param callable|None handle_monitor_message: a function to handle monitor messages (e.g. send them to an endpoint for plotting or displaying) - this function should take a single JSON-compatible python primitive :param bool record_events: if `True`, record received events in the `received_events` attribute :param dict|None event_handlers: a mapping of event type names to callables that handle each type of event. The handlers should not mutate the events. @@ -23,7 +23,7 @@ class EventReplayer(AbstractEventHandler): def __init__( self, - receiving_service=None, + recipient=None, handle_monitor_message=None, record_events=True, event_handlers=None, @@ -31,7 +31,7 @@ def __init__( only_handle_result=False, ): super().__init__( - receiving_service or Service(backend=ServiceBackend(), service_id="local/local:local"), + recipient or Service(backend=ServiceBackend(), service_id="local/local:local"), handle_monitor_message=handle_monitor_message, record_events=record_events, event_handlers=event_handlers, diff --git a/octue/cloud/events/validation.py b/octue/cloud/events/validation.py index d8cd1d802..39dfd003f 100644 --- a/octue/cloud/events/validation.py +++ b/octue/cloud/events/validation.py @@ -18,12 +18,12 @@ jsonschema_validator = jsonschema.Draft202012Validator(SERVICE_COMMUNICATION_SCHEMA) -def is_event_valid(event, attributes, receiving_service, parent_sdk_version, child_sdk_version, schema=None): +def is_event_valid(event, attributes, recipient, parent_sdk_version, child_sdk_version, schema=None): """Check if the event and its attributes are valid according to the Octue services communication schema. :param dict event: the event to validate :param dict attributes: the attributes of the event to validate - :param octue.cloud.pub_sub.service.Service receiving_service: the service receiving and validating the event + :param octue.cloud.pub_sub.service.Service recipient: the service receiving and validating the event :param str parent_sdk_version: the semantic version of Octue SDK running on the parent :param str child_sdk_version: the semantic version of Octue SDK running on the child :param dict|None schema: the schema to validate the event and its attributes against; if `None`, this defaults to the service communication schema used in this version of Octue SDK @@ -33,7 +33,7 @@ def is_event_valid(event, attributes, receiving_service, parent_sdk_version, chi raise_if_event_is_invalid( event, attributes, - receiving_service, + recipient, parent_sdk_version, child_sdk_version, schema=schema, @@ -44,19 +44,12 @@ def is_event_valid(event, attributes, receiving_service, parent_sdk_version, chi return True -def raise_if_event_is_invalid( - event, - attributes, - receiving_service, - parent_sdk_version, - child_sdk_version, - schema=None, -): +def raise_if_event_is_invalid(event, attributes, recipient, parent_sdk_version, child_sdk_version, schema=None): """Raise an error if the event or its attributes aren't valid according to the Octue services communication schema. :param dict event: the event to validate :param dict attributes: the attributes of the event to validate - :param octue.cloud.pub_sub.service.Service receiving_service: the service receiving and validating the event + :param octue.cloud.pub_sub.service.Service recipient: the service receiving and validating the event :param str parent_sdk_version: the semantic version of Octue SDK running on the parent :param str child_sdk_version: the semantic version of Octue SDK running on the child :param dict|None schema: the schema to validate the event and its attributes against; if `None`, this defaults to the service communication schema used in this version of Octue SDK @@ -82,7 +75,7 @@ def raise_if_event_is_invalid( logger.exception( "%r received an event that doesn't conform with version %s of the service communication schema (%s): %r.", - receiving_service, + recipient, SERVICE_COMMUNICATION_SCHEMA_VERSION, SERVICE_COMMUNICATION_SCHEMA_INFO_URL, event, diff --git a/octue/cloud/pub_sub/events.py b/octue/cloud/pub_sub/events.py index 8aa8731ba..115d8602f 100644 --- a/octue/cloud/pub_sub/events.py +++ b/octue/cloud/pub_sub/events.py @@ -65,7 +65,7 @@ class GoogleCloudPubSubEventHandler(AbstractEventHandler): """A synchronous handler for events received as Google Pub/Sub messages from a pull subscription. :param octue.cloud.pub_sub.subscription.Subscription subscription: the subscription messages are pulled from - :param octue.cloud.pub_sub.service.Service receiving_service: the service that's receiving the events + :param octue.cloud.pub_sub.service.Service recipient: the service that's receiving the events :param callable|None handle_monitor_message: a function to handle monitor messages (e.g. send them to an endpoint for plotting or displaying) - this function should take a single JSON-compatible python primitive :param bool record_events: if `True`, record received events in the `received_events` attribute :param dict|None event_handlers: a mapping of event type names to callables that handle each type of event. The handlers should not mutate the events. @@ -77,7 +77,7 @@ class GoogleCloudPubSubEventHandler(AbstractEventHandler): def __init__( self, subscription, - receiving_service, + recipient, handle_monitor_message=None, record_events=True, event_handlers=None, @@ -87,7 +87,7 @@ def __init__( self.subscription = subscription super().__init__( - receiving_service, + recipient, handle_monitor_message=handle_monitor_message, record_events=record_events, event_handlers=event_handlers, diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index 079fdd67b..dfa173bae 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -405,7 +405,7 @@ def wait_for_answer( self._event_handler = GoogleCloudPubSubEventHandler( subscription=subscription, - receiving_service=self, + recipient=self, handle_monitor_message=handle_monitor_message, record_events=record_messages, ) @@ -619,7 +619,7 @@ def _parse_question(self, question): raise_if_event_is_invalid( event=event_for_validation, attributes=attributes, - receiving_service=self, + recipient=self, parent_sdk_version=attributes["sender_sdk_version"], child_sdk_version=importlib.metadata.version("octue"), ) diff --git a/tests/cloud/pub_sub/test_events.py b/tests/cloud/pub_sub/test_events.py index 1ffb60025..0595a62bf 100644 --- a/tests/cloud/pub_sub/test_events.py +++ b/tests/cloud/pub_sub/test_events.py @@ -45,7 +45,7 @@ def test_timeout(self): """Test that a TimeoutError is raised if message handling takes longer than the given timeout.""" event_handler = GoogleCloudPubSubEventHandler( subscription=self.subscription, - receiving_service=self.parent, + recipient=self.parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: message}, schema={}, ) @@ -57,7 +57,7 @@ def test_in_order_messages_are_handled_in_order(self): """Test that messages received in order are handled in order.""" event_handler = GoogleCloudPubSubEventHandler( subscription=self.subscription, - receiving_service=self.parent, + recipient=self.parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, schema={}, ) @@ -109,7 +109,7 @@ def test_out_of_order_messages_are_handled_in_order(self): """Test that messages received out of order are handled in order.""" event_handler = GoogleCloudPubSubEventHandler( subscription=self.subscription, - receiving_service=self.parent, + recipient=self.parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, schema={}, ) @@ -164,7 +164,7 @@ def test_out_of_order_messages_with_end_message_first_are_handled_in_order(self) """ event_handler = GoogleCloudPubSubEventHandler( subscription=self.subscription, - receiving_service=self.parent, + recipient=self.parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, schema={}, ) @@ -217,7 +217,7 @@ def test_no_timeout(self): """Test that message handling works with no timeout.""" event_handler = GoogleCloudPubSubEventHandler( subscription=self.subscription, - receiving_service=self.parent, + recipient=self.parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, schema={}, ) @@ -258,7 +258,7 @@ def test_no_timeout(self): def test_delivery_acknowledgement(self): """Test that a delivery acknowledgement message is handled correctly.""" - event_handler = GoogleCloudPubSubEventHandler(subscription=self.subscription, receiving_service=self.parent) + event_handler = GoogleCloudPubSubEventHandler(subscription=self.subscription, recipient=self.parent) child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) messages = [ @@ -290,7 +290,7 @@ def test_delivery_acknowledgement(self): def test_error_raised_if_heartbeat_not_received_before_checked(self): """Test that an error is raised if a heartbeat isn't received before a heartbeat is first checked for.""" - event_handler = GoogleCloudPubSubEventHandler(subscription=self.subscription, receiving_service=self.parent) + event_handler = GoogleCloudPubSubEventHandler(subscription=self.subscription, recipient=self.parent) with self.assertRaises(TimeoutError) as error: event_handler.handle_events(maximum_heartbeat_interval=0) @@ -300,7 +300,7 @@ def test_error_raised_if_heartbeat_not_received_before_checked(self): def test_error_raised_if_heartbeats_stop_being_received(self): """Test that an error is raised if heartbeats stop being received within the maximum interval.""" - event_handler = GoogleCloudPubSubEventHandler(subscription=self.subscription, receiving_service=self.parent) + event_handler = GoogleCloudPubSubEventHandler(subscription=self.subscription, recipient=self.parent) event_handler._last_heartbeat = datetime.datetime.now() - datetime.timedelta(seconds=30) with self.assertRaises(TimeoutError) as error: @@ -310,7 +310,7 @@ def test_error_raised_if_heartbeats_stop_being_received(self): def test_error_not_raised_if_heartbeat_has_been_received_in_maximum_allowed_interval(self): """Test that an error is not raised if a heartbeat has been received in the maximum allowed interval.""" - event_handler = GoogleCloudPubSubEventHandler(subscription=self.subscription, receiving_service=self.parent) + event_handler = GoogleCloudPubSubEventHandler(subscription=self.subscription, recipient=self.parent) child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) event_handler._last_heartbeat = datetime.datetime.now() @@ -346,19 +346,19 @@ def test_error_not_raised_if_heartbeat_has_been_received_in_maximum_allowed_inte def test_time_since_last_heartbeat_is_none_if_no_heartbeat_received_yet(self): """Test that the time since the last heartbeat is `None` if no heartbeat has been received yet.""" - event_handler = GoogleCloudPubSubEventHandler(subscription=self.subscription, receiving_service=self.parent) + event_handler = GoogleCloudPubSubEventHandler(subscription=self.subscription, recipient=self.parent) self.assertIsNone(event_handler._time_since_last_heartbeat) def test_total_run_time_is_none_if_handle_events_has_not_been_called(self): """Test that the total run time for the message handler is `None` if the `handle_events` method has not been called. """ - event_handler = GoogleCloudPubSubEventHandler(subscription=self.subscription, receiving_service=self.parent) + event_handler = GoogleCloudPubSubEventHandler(subscription=self.subscription, recipient=self.parent) self.assertIsNone(event_handler.total_run_time) def test_time_since_missing_message_is_none_if_no_unhandled_missing_messages(self): """Test that the `time_since_missing_message` property is `None` if there are no unhandled missing messages.""" - event_handler = GoogleCloudPubSubEventHandler(subscription=self.subscription, receiving_service=self.parent) + event_handler = GoogleCloudPubSubEventHandler(subscription=self.subscription, recipient=self.parent) self.assertIsNone(event_handler.time_since_missing_event) def test_missing_messages_at_start_can_be_skipped(self): @@ -367,7 +367,7 @@ def test_missing_messages_at_start_can_be_skipped(self): """ event_handler = GoogleCloudPubSubEventHandler( subscription=self.subscription, - receiving_service=self.parent, + recipient=self.parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, schema={}, skip_missing_events_after=0, @@ -421,7 +421,7 @@ def test_missing_messages_in_middle_can_skipped(self): """Test that missing messages in the middle of the event stream can be skipped.""" event_handler = GoogleCloudPubSubEventHandler( subscription=self.subscription, - receiving_service=self.parent, + recipient=self.parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, schema={}, skip_missing_events_after=0, @@ -481,7 +481,7 @@ def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): """Test that multiple blocks of missing messages in the middle of the event stream can be skipped.""" event_handler = GoogleCloudPubSubEventHandler( subscription=self.subscription, - receiving_service=self.parent, + recipient=self.parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, schema={}, skip_missing_events_after=0, @@ -575,7 +575,7 @@ def test_all_messages_missing_apart_from_result(self): """Test that the result message is still handled if all other messages are missing.""" event_handler = GoogleCloudPubSubEventHandler( subscription=self.subscription, - receiving_service=self.parent, + recipient=self.parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, schema={}, skip_missing_events_after=0, @@ -638,7 +638,7 @@ def test_pull_and_enqueue_available_events(self): event_handler = GoogleCloudPubSubEventHandler( subscription=self.subscription, - receiving_service=self.parent, + recipient=self.parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, schema={}, ) @@ -679,7 +679,7 @@ def test_timeout_error_raised_if_result_message_not_received_in_time(self): event_handler = GoogleCloudPubSubEventHandler( subscription=self.subscription, - receiving_service=self.parent, + recipient=self.parent, event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, ) From 242d0971f8125e674cd8d484ffce32c1300d880f Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 9 Apr 2024 16:07:14 +0100 Subject: [PATCH 123/169] REF: Rename `Service._send_message` to `Service._emit_event` --- octue/cloud/events/counter.py | 2 +- octue/cloud/pub_sub/logging.py | 8 ++--- octue/cloud/pub_sub/service.py | 26 +++++++-------- tests/cloud/pub_sub/test_events.py | 52 ++++++++++++++--------------- tests/cloud/pub_sub/test_logging.py | 4 +-- 5 files changed, 46 insertions(+), 46 deletions(-) diff --git a/octue/cloud/events/counter.py b/octue/cloud/events/counter.py index 92cec1d7a..5edc208a1 100644 --- a/octue/cloud/events/counter.py +++ b/octue/cloud/events/counter.py @@ -1,6 +1,6 @@ class EventCounter: """A mutable counter for keeping track of the emission order of events. This is used in the `Service` class instead - of an integer because it is mutable and can be passed to the `Service._send_message` method and incremented as + of an integer because it is mutable and can be passed to the `Service._emit_event` method and incremented as events are emitted. :return None: diff --git a/octue/cloud/pub_sub/logging.py b/octue/cloud/pub_sub/logging.py index 80f9a744d..e8d78a6a8 100644 --- a/octue/cloud/pub_sub/logging.py +++ b/octue/cloud/pub_sub/logging.py @@ -8,7 +8,7 @@ class GoogleCloudPubSubHandler(logging.Handler): """A log handler that publishes log records to a Google Cloud Pub/Sub topic. - :param callable message_sender: the `_send_message` method of the service that instantiated this instance + :param callable event_emitter: the `_emit_event` method of the service that instantiated this instance :param str question_uuid: the UUID of the question to handle log records for :param str originator: the SRUID of the service that asked the question these log records are related to :param str recipient: the SRUID of the service to send these log records to @@ -17,14 +17,14 @@ class GoogleCloudPubSubHandler(logging.Handler): :return None: """ - def __init__(self, message_sender, question_uuid, originator, recipient, order, timeout=60, *args, **kwargs): + def __init__(self, event_emitter, question_uuid, originator, recipient, order, timeout=60, *args, **kwargs): super().__init__(*args, **kwargs) self.question_uuid = question_uuid self.originator = originator self.recipient = recipient self.order = order self.timeout = timeout - self._send_message = message_sender + self._emit_event = event_emitter def emit(self, record): """Serialise the log record as a dictionary and publish it to the topic. @@ -33,7 +33,7 @@ def emit(self, record): :return None: """ try: - self._send_message( + self._emit_event( { "kind": "log_record", "log_record": self._convert_log_record_to_primitives(record), diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index dfa173bae..b7991bb16 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -241,7 +241,7 @@ def answer(self, question, heartbeat_interval=120, timeout=30): if forward_logs: analysis_log_handler = GoogleCloudPubSubHandler( - message_sender=self._send_message, + event_emitter=self.emit_event, question_uuid=question_uuid, originator=originator, recipient=originator, @@ -270,8 +270,8 @@ def answer(self, question, heartbeat_interval=120, timeout=30): if analysis.output_manifest is not None: result["output_manifest"] = analysis.output_manifest.to_primitive() - self._send_message( - message=result, + self.emit_event( + event=result, originator=originator, recipient=originator, order=order, @@ -431,7 +431,7 @@ def send_exception(self, question_uuid, originator, order, timeout=30): exception = convert_exception_to_primitives() exception_message = f"Error in {self!r}: {exception['message']}" - self._send_message( + self.emit_event( { "kind": "exception", "exception_type": exception["type"], @@ -445,11 +445,11 @@ def send_exception(self, question_uuid, originator, order, timeout=30): timeout=timeout, ) - def _send_message(self, message, originator, recipient, order, attributes=None, timeout=30): - """Send a JSON-serialised event as a Pub/Sub message to the services topic with optional message attributes, + def emit_event(self, event, originator, recipient, order, attributes=None, timeout=30): + """Emit a JSON-serialised event as a Pub/Sub message to the services topic with optional message attributes, incrementing the `order` argument by one. This method is thread-safe. - :param dict message: JSON-serialisable data to send as a message + :param dict event: JSON-serialisable data to emit as an event :param str originator: the SRUID of the service that asked the question this event is related to :param str recipient: the SRUID of the service the event is intended for :param octue.cloud.events.counter.EventCounter order: an event counter keeping track of the order of emitted events @@ -478,7 +478,7 @@ def _send_message(self, message, originator, recipient, order, attributes=None, future = self.publisher.publish( topic=self.services_topic.path, - data=json.dumps(message, cls=OctueJSONEncoder).encode(), + data=json.dumps(event, cls=OctueJSONEncoder).encode(), retry=retry.Retry(deadline=timeout), **converted_attributes, ) @@ -516,8 +516,8 @@ def _send_question( input_manifest.use_signed_urls_for_datasets() question["input_manifest"] = input_manifest.to_primitive() - future = self._send_message( - message=question, + future = self.emit_event( + event=question, timeout=timeout, originator=self.id, recipient=service_id, @@ -543,7 +543,7 @@ def _send_delivery_acknowledgment(self, question_uuid, originator, order, timeou :param float timeout: time in seconds after which to give up sending :return None: """ - self._send_message( + self.emit_event( { "kind": "delivery_acknowledgement", "datetime": datetime.datetime.utcnow().isoformat(), @@ -566,7 +566,7 @@ def _send_heartbeat(self, question_uuid, originator, order, timeout=30): :param float timeout: time in seconds after which to give up sending :return None: """ - self._send_message( + self.emit_event( { "kind": "heartbeat", "datetime": datetime.datetime.utcnow().isoformat(), @@ -590,7 +590,7 @@ def _send_monitor_message(self, data, question_uuid, originator, order, timeout= :param float timeout: time in seconds to retry sending the message :return None: """ - self._send_message( + self.emit_event( {"kind": "monitor_message", "data": data}, originator=originator, recipient=originator, diff --git a/tests/cloud/pub_sub/test_events.py b/tests/cloud/pub_sub/test_events.py index 0595a62bf..f85ac5808 100644 --- a/tests/cloud/pub_sub/test_events.py +++ b/tests/cloud/pub_sub/test_events.py @@ -84,8 +84,8 @@ def test_in_order_messages_are_handled_in_order(self): ] for message in messages: - child._send_message( - message=message["event"], + child.emit_event( + event=message["event"], attributes=message["attributes"], originator=self.parent.id, recipient=self.parent.id, @@ -136,8 +136,8 @@ def test_out_of_order_messages_are_handled_in_order(self): ] for message in messages: - child._send_message( - message=message["event"], + child.emit_event( + event=message["event"], attributes=message["attributes"], originator=self.parent.id, recipient=self.parent.id, @@ -191,8 +191,8 @@ def test_out_of_order_messages_with_end_message_first_are_handled_in_order(self) ] for message in messages: - child._send_message( - message=message["event"], + child.emit_event( + event=message["event"], attributes=message["attributes"], originator=self.parent.id, recipient=self.parent.id, @@ -240,8 +240,8 @@ def test_no_timeout(self): ] for message in messages: - child._send_message( - message=message["event"], + child.emit_event( + event=message["event"], attributes=message["attributes"], originator=self.parent.id, recipient=self.parent.id, @@ -277,8 +277,8 @@ def test_delivery_acknowledgement(self): ] for message in messages: - child._send_message( - message=message["event"], + child.emit_event( + event=message["event"], attributes=message["attributes"], originator=self.parent.id, recipient=self.parent.id, @@ -330,8 +330,8 @@ def test_error_not_raised_if_heartbeat_has_been_received_in_maximum_allowed_inte ] for message in messages: - child._send_message( - message=message["event"], + child.emit_event( + event=message["event"], attributes=message["attributes"], originator=self.parent.id, recipient=self.parent.id, @@ -396,8 +396,8 @@ def test_missing_messages_at_start_can_be_skipped(self): ] for message in messages: - child._send_message( - message=message["event"], + child.emit_event( + event=message["event"], attributes=message["attributes"], originator=self.parent.id, recipient=self.parent.id, @@ -446,8 +446,8 @@ def test_missing_messages_in_middle_can_skipped(self): ] for message in messages: - child._send_message( - message=message["event"], + child.emit_event( + event=message["event"], attributes=message["attributes"], originator=self.parent.id, recipient=self.parent.id, @@ -455,8 +455,8 @@ def test_missing_messages_in_middle_can_skipped(self): ) # Send a final message. - child._send_message( - message={"kind": "finish-test", "order": 5}, + child.emit_event( + event={"kind": "finish-test", "order": 5}, attributes={"question_uuid": self.question_uuid, "sender_type": "CHILD"}, originator=self.parent.id, recipient=self.parent.id, @@ -506,8 +506,8 @@ def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): ] for message in messages: - child._send_message( - message=message["event"], + child.emit_event( + event=message["event"], attributes=message["attributes"], originator=self.parent.id, recipient=self.parent.id, @@ -515,8 +515,8 @@ def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): ) # Send another message. - child._send_message( - message={"kind": "test", "order": 5}, + child.emit_event( + event={"kind": "test", "order": 5}, attributes={"order": 5, "question_uuid": self.question_uuid, "sender_type": "CHILD"}, originator=self.parent.id, recipient=self.parent.id, @@ -545,8 +545,8 @@ def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): ] for message in messages: - child._send_message( - message=message["event"], + child.emit_event( + event=message["event"], attributes=message["attributes"], originator=self.parent.id, recipient=self.parent.id, @@ -584,8 +584,8 @@ def test_all_messages_missing_apart_from_result(self): child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) # Send the result message. - child._send_message( - message={"kind": "finish-test", "order": 1000}, + child.emit_event( + event={"kind": "finish-test", "order": 1000}, attributes={"question_uuid": self.question_uuid, "sender_type": "CHILD"}, originator=self.parent.id, recipient=self.parent.id, diff --git a/tests/cloud/pub_sub/test_logging.py b/tests/cloud/pub_sub/test_logging.py index c60b7bca2..1c9b5f0ce 100644 --- a/tests/cloud/pub_sub/test_logging.py +++ b/tests/cloud/pub_sub/test_logging.py @@ -45,7 +45,7 @@ def test_emit(self): service = MockService(backend=GCPPubSubBackend(project_name="blah")) GoogleCloudPubSubHandler( - message_sender=service._send_message, + event_emitter=service.emit_event, question_uuid=question_uuid, originator="another/service:1.0.0", recipient="another/service:1.0.0", @@ -75,7 +75,7 @@ def test_emit_with_non_json_serialisable_args(self): with patch("octue.cloud.emulators._pub_sub.MockPublisher.publish") as mock_publish: GoogleCloudPubSubHandler( - message_sender=service._send_message, + event_emitter=service.emit_event, question_uuid="question-uuid", originator="another/service:1.0.0", recipient="another/service:1.0.0", From 4a5ba60aea4e3991fc3fa48989b736a9f13c5b1d Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 9 Apr 2024 16:32:21 +0100 Subject: [PATCH 124/169] REF: Make services prefix a constant --- octue/__init__.py | 1 - octue/cli.py | 10 ---------- octue/cloud/events/__init__.py | 1 + octue/cloud/pub_sub/__init__.py | 5 ++--- octue/cloud/pub_sub/service.py | 9 +++++---- octue/configuration.py | 5 ----- octue/resources/service_backends.py | 18 ++++++------------ tests/cloud/emulators/test_child_emulator.py | 5 +++-- tests/cloud/pub_sub/test_events.py | 5 +++-- tests/cloud/pub_sub/test_logging.py | 3 ++- tests/cloud/pub_sub/test_service.py | 3 ++- tests/resources/test_child.py | 3 ++- tests/resources/test_service_backends.py | 17 +++++------------ tests/test_cli.py | 3 ++- 14 files changed, 33 insertions(+), 55 deletions(-) diff --git a/octue/__init__.py b/octue/__init__.py index a2acacf99..eea4e34fb 100644 --- a/octue/__init__.py +++ b/octue/__init__.py @@ -7,7 +7,6 @@ __all__ = ("Runner",) -DEFAULT_OCTUE_SERVICES_NAMESPACE = "octue.services" REPOSITORY_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) diff --git a/octue/cli.py b/octue/cli.py index 985c25f5b..08889808f 100644 --- a/octue/cli.py +++ b/octue/cli.py @@ -10,7 +10,6 @@ import click from google import auth -from octue import DEFAULT_OCTUE_SERVICES_NAMESPACE from octue.cloud import pub_sub, storage from octue.cloud.pub_sub.service import Service from octue.cloud.service_id import create_sruid, get_sruid_parts @@ -358,13 +357,6 @@ def deploy(): help="The service revision tag (e.g. 1.0.7). If this option isn't given, a random 'cool name' tag is generated e.g" ". 'curious-capybara'.", ) -@click.option( - "--services-namespace", - is_flag=False, - default=DEFAULT_OCTUE_SERVICES_NAMESPACE, - show_default=True, - help="The services namespace to subscribe to.", -) def create_push_subscription( project_name, service_namespace, @@ -372,7 +364,6 @@ def create_push_subscription( push_endpoint, expiration_time, revision_tag, - services_namespace, ): """Create a Google Pub/Sub push subscription for an Octue service for it to receive questions from parents. The subscription name is printed on completion. @@ -394,7 +385,6 @@ def create_push_subscription( push_endpoint, expiration_time=expiration_time, subscription_filter=f'attributes.recipient = "{sruid}" AND attributes.sender_type = "PARENT"', - services_namespace=services_namespace, ) click.echo(sruid) diff --git a/octue/cloud/events/__init__.py b/octue/cloud/events/__init__.py index e69de29bb..66898c235 100644 --- a/octue/cloud/events/__init__.py +++ b/octue/cloud/events/__init__.py @@ -0,0 +1 @@ +OCTUE_SERVICES_PREFIX = "octue.services" diff --git a/octue/cloud/pub_sub/__init__.py b/octue/cloud/pub_sub/__init__.py index 1b9f63f5e..b6239757a 100644 --- a/octue/cloud/pub_sub/__init__.py +++ b/octue/cloud/pub_sub/__init__.py @@ -1,3 +1,4 @@ +from octue.cloud.events import OCTUE_SERVICES_PREFIX from octue.cloud.service_id import convert_service_id_to_pub_sub_form from .subscription import Subscription @@ -13,7 +14,6 @@ def create_push_subscription( push_endpoint, subscription_filter=None, expiration_time=None, - services_namespace="octue.services", ): """Create a Google Pub/Sub push subscription for an Octue service for it to receive questions from parents. If a corresponding topic doesn't exist, it will be created first. @@ -23,7 +23,6 @@ def create_push_subscription( :param str push_endpoint: the HTTP/HTTPS endpoint of the service to push to. It should be fully formed and include the 'https://' prefix :param str|None subscription_filter: if specified, the filter to apply to the subscription; otherwise, no filter is applied :param float|None expiration_time: the number of seconds of inactivity after which the subscription should expire. If not provided, no expiration time is applied to the subscription - :param str services_namespace: the services namespace to subscribe to :return None: """ if expiration_time: @@ -33,7 +32,7 @@ def create_push_subscription( subscription = Subscription( name=convert_service_id_to_pub_sub_form(sruid), - topic=Topic(name=services_namespace, project_name=project_name), + topic=Topic(name=OCTUE_SERVICES_PREFIX, project_name=project_name), filter=subscription_filter, expiration_time=expiration_time, push_endpoint=push_endpoint, diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index b7991bb16..0a5dcfa55 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -14,6 +14,7 @@ from google.cloud import pubsub_v1 import octue.exceptions +from octue.cloud.events import OCTUE_SERVICES_PREFIX from octue.cloud.events.counter import EventCounter from octue.cloud.events.validation import raise_if_event_is_invalid from octue.cloud.pub_sub import Subscription, Topic @@ -124,10 +125,10 @@ def services_topic(self): :return octue.cloud.pub_sub.topic.Topic: the Octue services topic for the project """ if not self._services_topic: - topic = Topic(name=self.backend.services_namespace, project_name=self.backend.project_name) + topic = Topic(name=OCTUE_SERVICES_PREFIX, project_name=self.backend.project_name) if not topic.exists(): - raise octue.exceptions.ServiceNotFound(f"Topic {self.backend.services_namespace!r} cannot be found.") + raise octue.exceptions.ServiceNotFound(f"Topic {topic.name!r} cannot be found.") self._services_topic = topic @@ -158,7 +159,7 @@ def serve(self, timeout=None, delete_topic_and_subscription_on_exit=False, allow logger.info("Starting %r.", self) subscription = Subscription( - name=".".join((self.backend.services_namespace, self._pub_sub_id)), + name=".".join((OCTUE_SERVICES_PREFIX, self._pub_sub_id)), topic=self.services_topic, filter=f'attributes.recipient = "{self.id}" AND attributes.sender_type = "{PARENT_SENDER_TYPE}"', expiration_time=None, @@ -354,7 +355,7 @@ def ask( pub_sub_id = convert_service_id_to_pub_sub_form(self.id) answer_subscription = Subscription( - name=".".join((self.backend.services_namespace, pub_sub_id, ANSWERS_NAMESPACE, question_uuid)), + name=".".join((OCTUE_SERVICES_PREFIX, pub_sub_id, ANSWERS_NAMESPACE, question_uuid)), topic=self.services_topic, filter=( f'attributes.recipient = "{self.id}" ' diff --git a/octue/configuration.py b/octue/configuration.py index 540815aa7..c5f487013 100644 --- a/octue/configuration.py +++ b/octue/configuration.py @@ -4,8 +4,6 @@ import yaml -from octue import DEFAULT_OCTUE_SERVICES_NAMESPACE - logger = logging.getLogger(__name__) @@ -20,7 +18,6 @@ class ServiceConfiguration: :param str|None app_configuration_path: the path to the app configuration file containing configuration data for the service; if this is `None`, the default application configuration is used :param str|None diagnostics_cloud_path: the path to a cloud directory to store diagnostics (this includes the configuration, input values and manifest, and logs) :param iter(dict)|None service_registries: the names and endpoints of the registries used to resolve service revisions when asking questions; these should be in priority order (highest priority first) - :param str services_namespace: the services namespace to emit and consume events from :param str|None directory: if provided, find the app source, twine, and app configuration relative to this directory :return None: """ @@ -34,7 +31,6 @@ def __init__( app_configuration_path=None, diagnostics_cloud_path=None, service_registries=None, - services_namespace=DEFAULT_OCTUE_SERVICES_NAMESPACE, directory=None, **kwargs, ): @@ -42,7 +38,6 @@ def __init__( self.namespace = namespace self.diagnostics_cloud_path = diagnostics_cloud_path self.service_registries = service_registries - self.services_namespace = services_namespace if directory: directory = os.path.abspath(directory) diff --git a/octue/resources/service_backends.py b/octue/resources/service_backends.py index 438b448ad..b0296ed21 100644 --- a/octue/resources/service_backends.py +++ b/octue/resources/service_backends.py @@ -39,25 +39,19 @@ class GCPPubSubBackend(ServiceBackend): """A dataclass containing the details needed to use Google Cloud Pub/Sub as a Service backend. :param str project_name: the name of the project to use for Pub/Sub - :param str services_namespace: the name of the topic to publish/subscribe events to/from :return None: """ - ERROR_MESSAGE = ( - "`project_name` and `services_namespace` must be specified for a service to connect to the correct service - " - "one of these is currently None." - ) - - # "octue.services" is repeated here to avoid a circular import. - def __init__(self, project_name, services_namespace="octue.services"): - if project_name is None or services_namespace is None: - raise exceptions.CloudLocationNotSpecified(self.ERROR_MESSAGE) + def __init__(self, project_name): + if project_name is None: + raise exceptions.CloudLocationNotSpecified( + "`project_name` must be specified for a service to connect to the correct service - received None." + ) self.project_name = project_name - self.services_namespace = services_namespace def __repr__(self): - return f"<{type(self).__name__}(project_name={self.project_name!r}, services_namespace={self.services_namespace!r})>" + return f"<{type(self).__name__}(project_name={self.project_name!r})>" AVAILABLE_BACKENDS = { diff --git a/tests/cloud/emulators/test_child_emulator.py b/tests/cloud/emulators/test_child_emulator.py index d2c6a4fa3..4c0af6a00 100644 --- a/tests/cloud/emulators/test_child_emulator.py +++ b/tests/cloud/emulators/test_child_emulator.py @@ -4,6 +4,7 @@ from octue.cloud import storage from octue.cloud.emulators._pub_sub import MockService, MockTopic from octue.cloud.emulators.child import ChildEmulator, ServicePatcher +from octue.cloud.events import OCTUE_SERVICES_PREFIX from octue.cloud.storage import GoogleCloudStorageClient from octue.resources import Manifest from octue.resources.service_backends import GCPPubSubBackend @@ -17,7 +18,7 @@ class TestChildEmulatorAsk(BaseTestCase): @classmethod def setUpClass(cls): - topic = MockTopic(name="octue.services", project_name=TEST_PROJECT_NAME) + topic = MockTopic(name=OCTUE_SERVICES_PREFIX, project_name=TEST_PROJECT_NAME) with ServicePatcher(): topic.create(allow_existing=True) @@ -306,7 +307,7 @@ class TestChildEmulatorJSONFiles(BaseTestCase): @classmethod def setUpClass(cls): - topic = MockTopic(name="octue.services", project_name=TEST_PROJECT_NAME) + topic = MockTopic(name=OCTUE_SERVICES_PREFIX, project_name=TEST_PROJECT_NAME) with ServicePatcher(): topic.create(allow_existing=True) diff --git a/tests/cloud/pub_sub/test_events.py b/tests/cloud/pub_sub/test_events.py index f85ac5808..06d6d5cec 100644 --- a/tests/cloud/pub_sub/test_events.py +++ b/tests/cloud/pub_sub/test_events.py @@ -5,6 +5,7 @@ from octue.cloud.emulators._pub_sub import MESSAGES, MockMessage, MockService, MockSubscription, MockTopic from octue.cloud.emulators.child import ServicePatcher +from octue.cloud.events import OCTUE_SERVICES_PREFIX from octue.cloud.pub_sub.events import GoogleCloudPubSubEventHandler from octue.resources.service_backends import GCPPubSubBackend from tests import TEST_PROJECT_NAME @@ -19,7 +20,7 @@ def setUpClass(cls): cls.service_patcher.start() cls.question_uuid = str(uuid.uuid4()) - cls.topic = MockTopic(name="octue.services", project_name=TEST_PROJECT_NAME) + cls.topic = MockTopic(name=OCTUE_SERVICES_PREFIX, project_name=TEST_PROJECT_NAME) cls.topic.create(allow_existing=True) cls.subscription = MockSubscription( @@ -607,7 +608,7 @@ def setUpClass(cls): cls.service_patcher.start() cls.question_uuid = str(uuid.uuid4()) - cls.topic = MockTopic(name="octue.services", project_name=TEST_PROJECT_NAME) + cls.topic = MockTopic(name=OCTUE_SERVICES_PREFIX, project_name=TEST_PROJECT_NAME) cls.topic.create(allow_existing=True) cls.subscription = MockSubscription( diff --git a/tests/cloud/pub_sub/test_logging.py b/tests/cloud/pub_sub/test_logging.py index 1c9b5f0ce..d89cde444 100644 --- a/tests/cloud/pub_sub/test_logging.py +++ b/tests/cloud/pub_sub/test_logging.py @@ -5,6 +5,7 @@ from octue.cloud.emulators._pub_sub import MESSAGES, MockService, MockTopic from octue.cloud.emulators.child import ServicePatcher +from octue.cloud.events import OCTUE_SERVICES_PREFIX from octue.cloud.events.counter import EventCounter from octue.cloud.pub_sub.logging import GoogleCloudPubSubHandler from octue.resources.service_backends import GCPPubSubBackend @@ -27,7 +28,7 @@ def setUpClass(cls): :return None: """ cls.service_patcher.start() - topic = MockTopic(name="octue.services", project_name=TEST_PROJECT_NAME) + topic = MockTopic(name=OCTUE_SERVICES_PREFIX, project_name=TEST_PROJECT_NAME) topic.create(allow_existing=True) @classmethod diff --git a/tests/cloud/pub_sub/test_service.py b/tests/cloud/pub_sub/test_service.py index f1ac55b20..e37f579f9 100644 --- a/tests/cloud/pub_sub/test_service.py +++ b/tests/cloud/pub_sub/test_service.py @@ -21,6 +21,7 @@ ) from octue.cloud.emulators.child import ServicePatcher from octue.cloud.emulators.cloud_storage import mock_generate_signed_url +from octue.cloud.events import OCTUE_SERVICES_PREFIX from octue.cloud.pub_sub.service import Service from octue.exceptions import InvalidMonitorMessage from octue.resources import Analysis, Datafile, Dataset, Manifest @@ -49,7 +50,7 @@ def setUpClass(cls): :return None: """ cls.service_patcher.start() - topic = MockTopic(name="octue.services", project_name=TEST_PROJECT_NAME) + topic = MockTopic(name=OCTUE_SERVICES_PREFIX, project_name=TEST_PROJECT_NAME) topic.create(allow_existing=True) @classmethod diff --git a/tests/resources/test_child.py b/tests/resources/test_child.py index b80877608..502dcc940 100644 --- a/tests/resources/test_child.py +++ b/tests/resources/test_child.py @@ -10,6 +10,7 @@ from octue.cloud.emulators._pub_sub import MockAnalysis, MockService, MockTopic from octue.cloud.emulators.child import ServicePatcher +from octue.cloud.events import OCTUE_SERVICES_PREFIX from octue.resources.child import Child from octue.resources.service_backends import GCPPubSubBackend from tests import MOCK_SERVICE_REVISION_TAG, TEST_PROJECT_NAME @@ -41,7 +42,7 @@ def setUpClass(cls): :return None: """ cls.service_patcher.start() - topic = MockTopic(name="octue.services", project_name=TEST_PROJECT_NAME) + topic = MockTopic(name=OCTUE_SERVICES_PREFIX, project_name=TEST_PROJECT_NAME) topic.create(allow_existing=True) @classmethod diff --git a/tests/resources/test_service_backends.py b/tests/resources/test_service_backends.py index d61ff997e..f385ae3c1 100644 --- a/tests/resources/test_service_backends.py +++ b/tests/resources/test_service_backends.py @@ -16,16 +16,9 @@ def test_existing_backend_can_be_retrieved(self): def test_repr(self): """Test the representation displays as expected.""" - self.assertEqual( - repr(GCPPubSubBackend(project_name="hello", services_namespace="world")), - "", - ) + self.assertEqual(repr(GCPPubSubBackend(project_name="hello")), "") - def test_error_raised_if_project_name_or_services_namespace_is_none(self): - """Test that an error is raised if the project name or services namespace aren't given during `GCPPubSubBackend` - instantiation. - """ - for project_name, services_namespace in (("hello", None), (None, "world")): - with self.subTest(project_name=project_name, services_namespace=services_namespace): - with self.assertRaises(CloudLocationNotSpecified): - GCPPubSubBackend(project_name=project_name, services_namespace=services_namespace) + def test_error_raised_if_project_name_is_none(self): + """Test that an error is raised if the project name isn't given during `GCPPubSubBackend` instantiation.""" + with self.assertRaises(CloudLocationNotSpecified): + GCPPubSubBackend(project_name=None) diff --git a/tests/test_cli.py b/tests/test_cli.py index 2b8f59028..a2e661d93 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -11,6 +11,7 @@ from octue.cloud import storage from octue.cloud.emulators._pub_sub import MockService, MockTopic from octue.cloud.emulators.child import ServicePatcher +from octue.cloud.events import OCTUE_SERVICES_PREFIX from octue.configuration import AppConfiguration, ServiceConfiguration from octue.resources import Dataset from octue.utils.patches import MultiPatcher @@ -194,7 +195,7 @@ def setUpClass(cls): }, ) - topic = MockTopic(name="octue.services", project_name=TEST_PROJECT_NAME) + topic = MockTopic(name=OCTUE_SERVICES_PREFIX, project_name=TEST_PROJECT_NAME) with ServicePatcher(): topic.create(allow_existing=True) From 0fcdc979579e0459f9419f403428f1f1eece49df Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 9 Apr 2024 16:39:05 +0100 Subject: [PATCH 125/169] REF: Rename `Service.emit_event` to `Service._emit_event` skipci --- octue/cloud/pub_sub/service.py | 16 ++++++------ .../cloud_run/test_cloud_run_deployment.py | 2 +- tests/cloud/pub_sub/test_events.py | 26 +++++++++---------- tests/cloud/pub_sub/test_logging.py | 4 +-- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index 0a5dcfa55..cdbcc443a 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -242,7 +242,7 @@ def answer(self, question, heartbeat_interval=120, timeout=30): if forward_logs: analysis_log_handler = GoogleCloudPubSubHandler( - event_emitter=self.emit_event, + event_emitter=self._emit_event, question_uuid=question_uuid, originator=originator, recipient=originator, @@ -271,7 +271,7 @@ def answer(self, question, heartbeat_interval=120, timeout=30): if analysis.output_manifest is not None: result["output_manifest"] = analysis.output_manifest.to_primitive() - self.emit_event( + self._emit_event( event=result, originator=originator, recipient=originator, @@ -432,7 +432,7 @@ def send_exception(self, question_uuid, originator, order, timeout=30): exception = convert_exception_to_primitives() exception_message = f"Error in {self!r}: {exception['message']}" - self.emit_event( + self._emit_event( { "kind": "exception", "exception_type": exception["type"], @@ -446,7 +446,7 @@ def send_exception(self, question_uuid, originator, order, timeout=30): timeout=timeout, ) - def emit_event(self, event, originator, recipient, order, attributes=None, timeout=30): + def _emit_event(self, event, originator, recipient, order, attributes=None, timeout=30): """Emit a JSON-serialised event as a Pub/Sub message to the services topic with optional message attributes, incrementing the `order` argument by one. This method is thread-safe. @@ -517,7 +517,7 @@ def _send_question( input_manifest.use_signed_urls_for_datasets() question["input_manifest"] = input_manifest.to_primitive() - future = self.emit_event( + future = self._emit_event( event=question, timeout=timeout, originator=self.id, @@ -544,7 +544,7 @@ def _send_delivery_acknowledgment(self, question_uuid, originator, order, timeou :param float timeout: time in seconds after which to give up sending :return None: """ - self.emit_event( + self._emit_event( { "kind": "delivery_acknowledgement", "datetime": datetime.datetime.utcnow().isoformat(), @@ -567,7 +567,7 @@ def _send_heartbeat(self, question_uuid, originator, order, timeout=30): :param float timeout: time in seconds after which to give up sending :return None: """ - self.emit_event( + self._emit_event( { "kind": "heartbeat", "datetime": datetime.datetime.utcnow().isoformat(), @@ -591,7 +591,7 @@ def _send_monitor_message(self, data, question_uuid, originator, order, timeout= :param float timeout: time in seconds to retry sending the message :return None: """ - self.emit_event( + self._emit_event( {"kind": "monitor_message", "data": data}, originator=originator, recipient=originator, diff --git a/tests/cloud/deployment/google/cloud_run/test_cloud_run_deployment.py b/tests/cloud/deployment/google/cloud_run/test_cloud_run_deployment.py index 4006e9333..1169e5f52 100644 --- a/tests/cloud/deployment/google/cloud_run/test_cloud_run_deployment.py +++ b/tests/cloud/deployment/google/cloud_run/test_cloud_run_deployment.py @@ -13,7 +13,7 @@ class TestCloudRunDeployment(TestCase): # This is the service ID of the example service deployed to Google Cloud Run. child = Child( - id="octue/example-service-cloud-run:0.3.2", + id="octue/example-service-cloud-run:stream-event", backend={"name": "GCPPubSubBackend", "project_name": os.environ["TEST_PROJECT_NAME"]}, ) diff --git a/tests/cloud/pub_sub/test_events.py b/tests/cloud/pub_sub/test_events.py index 06d6d5cec..4e75fdbb9 100644 --- a/tests/cloud/pub_sub/test_events.py +++ b/tests/cloud/pub_sub/test_events.py @@ -85,7 +85,7 @@ def test_in_order_messages_are_handled_in_order(self): ] for message in messages: - child.emit_event( + child._emit_event( event=message["event"], attributes=message["attributes"], originator=self.parent.id, @@ -137,7 +137,7 @@ def test_out_of_order_messages_are_handled_in_order(self): ] for message in messages: - child.emit_event( + child._emit_event( event=message["event"], attributes=message["attributes"], originator=self.parent.id, @@ -192,7 +192,7 @@ def test_out_of_order_messages_with_end_message_first_are_handled_in_order(self) ] for message in messages: - child.emit_event( + child._emit_event( event=message["event"], attributes=message["attributes"], originator=self.parent.id, @@ -241,7 +241,7 @@ def test_no_timeout(self): ] for message in messages: - child.emit_event( + child._emit_event( event=message["event"], attributes=message["attributes"], originator=self.parent.id, @@ -278,7 +278,7 @@ def test_delivery_acknowledgement(self): ] for message in messages: - child.emit_event( + child._emit_event( event=message["event"], attributes=message["attributes"], originator=self.parent.id, @@ -331,7 +331,7 @@ def test_error_not_raised_if_heartbeat_has_been_received_in_maximum_allowed_inte ] for message in messages: - child.emit_event( + child._emit_event( event=message["event"], attributes=message["attributes"], originator=self.parent.id, @@ -397,7 +397,7 @@ def test_missing_messages_at_start_can_be_skipped(self): ] for message in messages: - child.emit_event( + child._emit_event( event=message["event"], attributes=message["attributes"], originator=self.parent.id, @@ -447,7 +447,7 @@ def test_missing_messages_in_middle_can_skipped(self): ] for message in messages: - child.emit_event( + child._emit_event( event=message["event"], attributes=message["attributes"], originator=self.parent.id, @@ -456,7 +456,7 @@ def test_missing_messages_in_middle_can_skipped(self): ) # Send a final message. - child.emit_event( + child._emit_event( event={"kind": "finish-test", "order": 5}, attributes={"question_uuid": self.question_uuid, "sender_type": "CHILD"}, originator=self.parent.id, @@ -507,7 +507,7 @@ def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): ] for message in messages: - child.emit_event( + child._emit_event( event=message["event"], attributes=message["attributes"], originator=self.parent.id, @@ -516,7 +516,7 @@ def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): ) # Send another message. - child.emit_event( + child._emit_event( event={"kind": "test", "order": 5}, attributes={"order": 5, "question_uuid": self.question_uuid, "sender_type": "CHILD"}, originator=self.parent.id, @@ -546,7 +546,7 @@ def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): ] for message in messages: - child.emit_event( + child._emit_event( event=message["event"], attributes=message["attributes"], originator=self.parent.id, @@ -585,7 +585,7 @@ def test_all_messages_missing_apart_from_result(self): child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) # Send the result message. - child.emit_event( + child._emit_event( event={"kind": "finish-test", "order": 1000}, attributes={"question_uuid": self.question_uuid, "sender_type": "CHILD"}, originator=self.parent.id, diff --git a/tests/cloud/pub_sub/test_logging.py b/tests/cloud/pub_sub/test_logging.py index d89cde444..2f26af05d 100644 --- a/tests/cloud/pub_sub/test_logging.py +++ b/tests/cloud/pub_sub/test_logging.py @@ -46,7 +46,7 @@ def test_emit(self): service = MockService(backend=GCPPubSubBackend(project_name="blah")) GoogleCloudPubSubHandler( - event_emitter=service.emit_event, + event_emitter=service._emit_event, question_uuid=question_uuid, originator="another/service:1.0.0", recipient="another/service:1.0.0", @@ -76,7 +76,7 @@ def test_emit_with_non_json_serialisable_args(self): with patch("octue.cloud.emulators._pub_sub.MockPublisher.publish") as mock_publish: GoogleCloudPubSubHandler( - event_emitter=service.emit_event, + event_emitter=service._emit_event, question_uuid="question-uuid", originator="another/service:1.0.0", recipient="another/service:1.0.0", From 7ccb02066a337e463daabdb8181221bb5dce1e8c Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 9 Apr 2024 16:51:58 +0100 Subject: [PATCH 126/169] TST: Test error is raised if service topic is missing --- tests/cloud/pub_sub/test_service.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/cloud/pub_sub/test_service.py b/tests/cloud/pub_sub/test_service.py index e37f579f9..a03ce9ec4 100644 --- a/tests/cloud/pub_sub/test_service.py +++ b/tests/cloud/pub_sub/test_service.py @@ -115,6 +115,14 @@ def test_serve_detached(self): self.assertFalse(future.returned) self.assertFalse(subscriber.closed) + def test_missing_services_topic_results_in_error(self): + """Test that an error is raised if the services topic doesn't exist.""" + service = MockService(backend=BACKEND) + + with patch("octue.cloud.emulators._pub_sub.TOPICS", set()): + with self.assertRaises(exceptions.ServiceNotFound): + service.services_topic + def test_ask_unregistered_service_revision_when_service_registries_specified_results_in_error(self): """Test that an error is raised if attempting to ask an unregistered service a question when service registries are being used. @@ -549,7 +557,7 @@ def test_service_can_ask_multiple_questions_to_child(self): for i in range(5): subscription, _ = parent.ask(service_id=child.id, input_values={}) - answers.append(parent.wait_for_answer(subscription, timeout=3600)) + answers.append(parent.wait_for_answer(subscription)) for answer in answers: self.assertEqual( From a54c04b141e17df09e600cd505337aed351626c7 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 9 Apr 2024 16:56:00 +0100 Subject: [PATCH 127/169] TST: Update cloud deployment test skipci --- .../deployment/google/cloud_run/test_cloud_run_deployment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cloud/deployment/google/cloud_run/test_cloud_run_deployment.py b/tests/cloud/deployment/google/cloud_run/test_cloud_run_deployment.py index 1169e5f52..bb6e12903 100644 --- a/tests/cloud/deployment/google/cloud_run/test_cloud_run_deployment.py +++ b/tests/cloud/deployment/google/cloud_run/test_cloud_run_deployment.py @@ -13,7 +13,7 @@ class TestCloudRunDeployment(TestCase): # This is the service ID of the example service deployed to Google Cloud Run. child = Child( - id="octue/example-service-cloud-run:stream-event", + id="octue/example-service-cloud-run:0.4.0", backend={"name": "GCPPubSubBackend", "project_name": os.environ["TEST_PROJECT_NAME"]}, ) From 18dcd7c4ea7e7091f165364af8521820992bb4f2 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 9 Apr 2024 18:05:23 +0100 Subject: [PATCH 128/169] DOC: Add missing word to docstring --- octue/cloud/emulators/_pub_sub.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octue/cloud/emulators/_pub_sub.py b/octue/cloud/emulators/_pub_sub.py index 527ceb45b..09e80f27c 100644 --- a/octue/cloud/emulators/_pub_sub.py +++ b/octue/cloud/emulators/_pub_sub.py @@ -243,7 +243,7 @@ def __init__(self, message): self.ack_id = None def __repr__(self): - """Represent the mock message as a string. + """Represent the mock message wrapper as a string. :return str: """ From a9981f5fed01848e8a292663cb590f8a20748f60 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 9 Apr 2024 18:06:07 +0100 Subject: [PATCH 129/169] CHO: Remove BigQuery subscription abilities --- octue/cloud/pub_sub/subscription.py | 24 +------------ octue/exceptions.py | 6 ---- tests/cloud/pub_sub/test_subscription.py | 44 ------------------------ 3 files changed, 1 insertion(+), 73 deletions(-) diff --git a/octue/cloud/pub_sub/subscription.py b/octue/cloud/pub_sub/subscription.py index c9920be03..53f06f6fd 100644 --- a/octue/cloud/pub_sub/subscription.py +++ b/octue/cloud/pub_sub/subscription.py @@ -5,7 +5,6 @@ from google.protobuf.duration_pb2 import Duration # noqa from google.protobuf.field_mask_pb2 import FieldMask # noqa from google.pubsub_v1.types.pubsub import ( - BigQueryConfig, ExpirationPolicy, PushConfig, RetryPolicy, @@ -13,8 +12,6 @@ UpdateSubscriptionRequest, ) -from octue.exceptions import ConflictingSubscriptionType - logger = logging.getLogger(__name__) @@ -36,7 +33,6 @@ class Subscription: :param float minimum_retry_backoff: minimum number of seconds after the acknowledgement deadline has passed to exponentially retry delivering a message to the subscription :param float maximum_retry_backoff: maximum number of seconds after the acknowledgement deadline has passed to exponentially retry delivering a message to the subscription :param str|None push_endpoint: if this is a push subscription, this is the URL to which messages should be pushed; leave as `None` if it's not a push subscription - :param str|None bigquery_table_id: if this is a BigQuery subscription, this is the ID of the table to which messages should be written (e.g. "your-project.your-dataset.your-table"); leave as `None` if it's not a BigQuery subscription :return None: """ @@ -52,7 +48,6 @@ def __init__( minimum_retry_backoff=10, maximum_retry_backoff=600, push_endpoint=None, - bigquery_table_id=None, ): self.name = name self.topic = topic @@ -73,14 +68,7 @@ def __init__( maximum_backoff=Duration(seconds=maximum_retry_backoff), ) - if push_endpoint and bigquery_table_id: - raise ConflictingSubscriptionType( - f"A subscription can only have one of `push_endpoint` and `bigquery_table_id`; received " - f"`push_endpoint={push_endpoint!r}` and `bigquery_table_id={bigquery_table_id!r}`." - ) - self.push_endpoint = push_endpoint - self.bigquery_table_id = bigquery_table_id self._subscriber = SubscriberClient() self._created = False @@ -99,7 +87,7 @@ def is_pull_subscription(self): :return bool: """ - return (self.push_endpoint is None) and (self.bigquery_table_id is None) + return self.push_endpoint is None @property def is_push_subscription(self): @@ -109,14 +97,6 @@ def is_push_subscription(self): """ return self.push_endpoint is not None - @property - def is_bigquery_subscription(self): - """Return `True` if this is a BigQuery subscription. - - :return bool: - """ - return self.bigquery_table_id is not None - def __repr__(self): """Represent the subscription as a string. @@ -199,8 +179,6 @@ def _create_proto_message_subscription(self): """ if self.push_endpoint: options = {"push_config": PushConfig(mapping=None, push_endpoint=self.push_endpoint)} # noqa - elif self.bigquery_table_id: - options = {"bigquery_config": BigQueryConfig(table=self.bigquery_table_id, write_metadata=True)} # noqa else: options = {} diff --git a/octue/exceptions.py b/octue/exceptions.py index 1e93ea037..93ee75479 100644 --- a/octue/exceptions.py +++ b/octue/exceptions.py @@ -118,11 +118,5 @@ class NotAPullSubscription(OctueSDKException): """Raise if attempting to pull a subscription that's not a pull subscription.""" -class ConflictingSubscriptionType(OctueSDKException): - """Raise if attempting to instantiate a subscription that's a push subscription and BigQuery subscription at the - same time. - """ - - class ReadOnlyResource(OctueSDKException): """Raise if attempting to alter a read-only resource.""" diff --git a/tests/cloud/pub_sub/test_subscription.py b/tests/cloud/pub_sub/test_subscription.py index bbf4961c8..649ab21f5 100644 --- a/tests/cloud/pub_sub/test_subscription.py +++ b/tests/cloud/pub_sub/test_subscription.py @@ -6,7 +6,6 @@ from octue.cloud.emulators._pub_sub import MockSubscriber, MockSubscriptionCreationResponse from octue.cloud.pub_sub.subscription import THIRTY_ONE_DAYS, Subscription from octue.cloud.pub_sub.topic import Topic -from octue.exceptions import ConflictingSubscriptionType from tests import TEST_PROJECT_NAME from tests.base import BaseTestCase @@ -75,21 +74,6 @@ def test_create_pull_subscription(self): self.assertEqual(response._pb.retry_policy.minimum_backoff.seconds, 10) self.assertEqual(response._pb.retry_policy.maximum_backoff.seconds, 600) - def test_error_raised_if_attempting_to_create_push_subscription_at_same_time_as_bigquery_subscription(self): - """Test that an error is raised if attempting to create a subscription that's both a push subscription and a - BigQuery subscription. - """ - project_name = os.environ["TEST_PROJECT_NAME"] - topic = Topic(name="my-topic", project_name=project_name) - - with self.assertRaises(ConflictingSubscriptionType): - Subscription( - name="world", - topic=topic, - push_endpoint="https://example.com/endpoint", - bigquery_table_id="my-project.my-dataset.my-table", - ) - def test_create_push_subscription(self): """Test that creating a push subscription works properly.""" project_name = os.environ["TEST_PROJECT_NAME"] @@ -106,41 +90,13 @@ def test_create_push_subscription(self): self.assertEqual(response._pb.retry_policy.maximum_backoff.seconds, 600) self.assertEqual(response._pb.push_config.push_endpoint, "https://example.com/endpoint") - def test_create_bigquery_subscription(self): - """Test that creating a BigQuery subscription works properly.""" - project_name = os.environ["TEST_PROJECT_NAME"] - topic = Topic(name="my-topic", project_name=project_name) - subscription = Subscription(name="world", topic=topic, bigquery_table_id="my-project.my-dataset.my-table") - - with patch("google.pubsub_v1.SubscriberClient.create_subscription", new=MockSubscriptionCreationResponse): - response = subscription.create(allow_existing=True) - - self.assertEqual(response._pb.ack_deadline_seconds, 600) - self.assertEqual(response._pb.expiration_policy.ttl.seconds, THIRTY_ONE_DAYS) - self.assertEqual(response._pb.message_retention_duration.seconds, 600) - self.assertEqual(response._pb.retry_policy.minimum_backoff.seconds, 10) - self.assertEqual(response._pb.retry_policy.maximum_backoff.seconds, 600) - self.assertEqual(response._pb.bigquery_config.table, "my-project.my-dataset.my-table") - self.assertTrue(response._pb.bigquery_config.write_metadata) - self.assertEqual(response._pb.push_config.push_endpoint, "") - def test_is_pull_subscription(self): """Test that `is_pull_subscription` is `True` for a pull subscription.""" self.assertTrue(self.subscription.is_pull_subscription) self.assertFalse(self.subscription.is_push_subscription) - self.assertFalse(self.subscription.is_bigquery_subscription) def test_is_push_subscription(self): """Test that `is_push_subscription` is `True` for a push subscription.""" push_subscription = Subscription(name="world", topic=self.topic, push_endpoint="https://example.com/endpoint") self.assertTrue(push_subscription.is_push_subscription) self.assertFalse(push_subscription.is_pull_subscription) - self.assertFalse(push_subscription.is_bigquery_subscription) - - def test_is_bigquery_subscription(self): - """Test that `is_bigquery_subscription` is `True` for a BigQuery subscription.""" - subscription = Subscription(name="world", topic=self.topic, bigquery_table_id="my-project.my-dataset.my-table") - - self.assertTrue(subscription.is_bigquery_subscription) - self.assertFalse(subscription.is_pull_subscription) - self.assertFalse(subscription.is_push_subscription) From 19dc2410161bb4a4ba15c8f56519f4ba47c06cc8 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 9 Apr 2024 18:10:14 +0100 Subject: [PATCH 130/169] DOC: Correct docstring --- octue/resources/child.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octue/resources/child.py b/octue/resources/child.py index 915490fa5..68a7a86c1 100644 --- a/octue/resources/child.py +++ b/octue/resources/child.py @@ -83,7 +83,7 @@ def ask( :param bool record_messages: if `True`, record messages received from the child in the `received_messages` property :param str save_diagnostics: must be one of {"SAVE_DIAGNOSTICS_OFF", "SAVE_DIAGNOSTICS_ON_CRASH", "SAVE_DIAGNOSTICS_ON"}; if turned on, allow the input values and manifest (and its datasets) to be saved by the child either all the time or just if it fails while processing them :param str|None question_uuid: the UUID to use for the question if a specific one is needed; a UUID is generated if not - :param str|None push_endpoint: if answers to the question should be pushed to an endpoint, provide its URL here (the returned subscription will be a push subscription); if not, leave this as `None` + :param str|None push_endpoint: if answers to the question should be pushed to an endpoint, provide its URL here; if not, leave this as `None` :param bool asynchronous: if `True`, don't create an answer subscription :param float timeout: time in seconds to wait for an answer before raising a timeout error :param float|int maximum_heartbeat_interval: the maximum amount of time (in seconds) allowed between child heartbeats before an error is raised From d73b0d386143eee20928f9a03cac12ed3f4bc271 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 10 Apr 2024 10:12:03 +0100 Subject: [PATCH 131/169] REF: Improve `AbstractEventHandler` docstring and rename attribute --- octue/cloud/events/handler.py | 25 +++++++++++++++---------- tests/cloud/pub_sub/test_events.py | 4 ++-- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/octue/cloud/events/handler.py b/octue/cloud/events/handler.py index 62c2d489f..d25dea2fe 100644 --- a/octue/cloud/events/handler.py +++ b/octue/cloud/events/handler.py @@ -28,16 +28,21 @@ class AbstractEventHandler: - """An abstract event handler. Inherit from this and add the `handle_events` and `_extract_event_and_attributes` - methods to handle events from a specific source synchronously or asynchronously. + """An abstract event handler for Octue service events that: + - Provide handlers for the Octue service event kinds (see https://strands.octue.com/octue/service-communication) + - Handles received events in the order specified by the `order` attribute + - Skips missing events after a set time and carries on handling from the next available event - :param octue.cloud.pub_sub.service.Service recipient: the service that's receiving the events + To create a concrete handler for a specific service/communication backend synchronously or asynchronously, inherit + from this class and add the `handle_events` and `_extract_event_and_attributes` methods. + + :param octue.cloud.pub_sub.service.Service recipient: the service instances that's receiving the events :param callable|None handle_monitor_message: a function to handle monitor messages (e.g. send them to an endpoint for plotting or displaying) - this function should take a single JSON-compatible python primitive :param bool record_events: if `True`, record received events in the `received_events` attribute - :param dict|None event_handlers: a mapping of event type names to callables that handle each type of event. The handlers should not mutate the events. - :param dict|str schema: the JSON schema (or URI of one) to validate events against + :param dict|None event_handlers: a mapping of event type names to callables that handle each type of event. The handlers must not mutate the events. + :param dict schema: the JSON schema to validate events against :param int|float skip_missing_events_after: the number of seconds after which to skip any events if they haven't arrived but subsequent events have - :param bool only_handle_result: if `True`, skip non-result events and only handle the result event + :param bool only_handle_result: if `True`, skip non-result events and only handle the result event when received :return None: """ @@ -59,8 +64,8 @@ def __init__( # These are set when the first event is received. self.question_uuid = None - self._child_sdk_version = None self.child_sruid = None + self.child_sdk_version = None self.waiting_events = None self.handled_events = [] @@ -130,10 +135,10 @@ def _extract_and_enqueue_event(self, container): return # Get the child's SRUID and Octue SDK version from the first event. - if not self._child_sdk_version: - self.child_sruid = attributes["sender"] + if not self.child_sdk_version: self.question_uuid = attributes["question_uuid"] - self._child_sdk_version = attributes["sender_sdk_version"] + self.child_sruid = attributes["sender"] + self.child_sdk_version = attributes["sender_sdk_version"] logger.debug("%r received an event related to question %r.", self.recipient, self.question_uuid) order = attributes["order"] diff --git a/tests/cloud/pub_sub/test_events.py b/tests/cloud/pub_sub/test_events.py index 4e75fdbb9..749e0bf02 100644 --- a/tests/cloud/pub_sub/test_events.py +++ b/tests/cloud/pub_sub/test_events.py @@ -646,7 +646,7 @@ def test_pull_and_enqueue_available_events(self): event_handler.question_uuid = self.question_uuid event_handler.child_sruid = "my-org/my-service:1.0.0" - event_handler._child_sdk_version = "0.1.3" + event_handler.child_sdk_version = "0.1.3" event_handler.waiting_events = {} # Enqueue a mock message for a mock subscription to receive. @@ -684,7 +684,7 @@ def test_timeout_error_raised_if_result_message_not_received_in_time(self): event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, ) - event_handler._child_sdk_version = "0.1.3" + event_handler.child_sdk_version = "0.1.3" event_handler.waiting_events = {} event_handler._start_time = 0 From 32d9ded5433930cb14516fcaf6f0711e3dd44c24 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 10 Apr 2024 10:34:20 +0100 Subject: [PATCH 132/169] REF: Improve clarity of `AbstractEventHandler` --- octue/cloud/events/handler.py | 59 +++++++++++++++++++++++------------ 1 file changed, 39 insertions(+), 20 deletions(-) diff --git a/octue/cloud/events/handler.py b/octue/cloud/events/handler.py index d25dea2fe..90994814e 100644 --- a/octue/cloud/events/handler.py +++ b/octue/cloud/events/handler.py @@ -42,7 +42,7 @@ class AbstractEventHandler: :param dict|None event_handlers: a mapping of event type names to callables that handle each type of event. The handlers must not mutate the events. :param dict schema: the JSON schema to validate events against :param int|float skip_missing_events_after: the number of seconds after which to skip any events if they haven't arrived but subsequent events have - :param bool only_handle_result: if `True`, skip non-result events and only handle the result event when received + :param bool only_handle_result: if `True`, skip non-result events and only handle the "result" event when received :return None: """ @@ -86,14 +86,22 @@ def __init__( self._log_message_colours = [COLOUR_PALETTE[1], *COLOUR_PALETTE[3:]] + @property + def awaiting_missing_event(self): + """Check if the event handler is currently waiting for a missing event. + + :return bool: `True` if the event handler is currently waiting for a missing event + """ + return self._missing_event_detection_time is not None + @property def time_since_missing_event(self): """Get the amount of time elapsed since the last missing event was detected. If no missing events have been - detected or they've already been skipped past, `None` is returned. + detected or they've already been skipped, `None` is returned. :return float|None: """ - if self._missing_event_detection_time is None: + if not self.awaiting_missing_event: return None return time.perf_counter() - self._missing_event_detection_time @@ -117,7 +125,7 @@ def _extract_event_and_attributes(self, container): pass def _extract_and_enqueue_event(self, container): - """Extract an event from its container and add it to `self.waiting_events`. + """Extract an event from its container, validate it, and add it to `self.waiting_events` if it's valid. :param any container: the container of the event (e.g. a Pub/Sub message) :return None: @@ -140,12 +148,12 @@ def _extract_and_enqueue_event(self, container): self.child_sruid = attributes["sender"] self.child_sdk_version = attributes["sender_sdk_version"] - logger.debug("%r received an event related to question %r.", self.recipient, self.question_uuid) + logger.debug("%r: Received an event related to question %r.", self.recipient, self.question_uuid) order = attributes["order"] if order in self.waiting_events: logger.warning( - "%r: Event with duplicate order %d received for question %s - overwriting original event.", + "%r: Event with duplicate order %d received for question %r - overwriting original event.", self.recipient, order, self.question_uuid, @@ -154,12 +162,12 @@ def _extract_and_enqueue_event(self, container): self.waiting_events[order] = event def _attempt_to_handle_waiting_events(self): - """Attempt to handle events waiting in `self.waiting_events`. If these events aren't consecutive to the + """Attempt to handle any events waiting in `self.waiting_events`. If these events aren't consecutive to the last handled event (i.e. if events have been received out of order and the next in-order event hasn't been received yet), just return. After the missing event wait time has passed, if this set of missing events haven't arrived but subsequent ones have, skip to the earliest waiting event and continue from there. - :return any|None: either a handled non-`None` result, or `None` if nothing was returned by the event handlers or if the next in-order event hasn't been received yet + :return any|None: either a handled non-`None` "result" event, or `None` if nothing was returned by the event handlers or if the next in-order event hasn't been received yet """ while self.waiting_events: try: @@ -169,7 +177,7 @@ def _attempt_to_handle_waiting_events(self): # If the next consecutive event hasn't been received: except KeyError: # Start the missing event timer if it isn't already running. - if self._missing_event_detection_time is None: + if not self.awaiting_missing_event: self._missing_event_detection_time = time.perf_counter() if self.time_since_missing_event > self.skip_missing_events_after: @@ -192,7 +200,7 @@ def _attempt_to_handle_waiting_events(self): def _skip_to_earliest_waiting_event(self): """Get the earliest waiting event and set the event handler up to continue from it. - :return dict|None: + :return dict|None: the earliest waiting event if there is one """ try: event = self.waiting_events.pop(self._earliest_waiting_event_number) @@ -219,8 +227,8 @@ def _skip_to_earliest_waiting_event(self): def _handle_event(self, event): """Pass an event to its handler and update the previous event number. - :param dict event: - :return dict|None: + :param dict event: the event to handle + :return dict|None: the output of the event (this should be `None` unless the event is a "result" event) """ self._previous_event_number += 1 @@ -248,22 +256,33 @@ def _handle_heartbeat(self, event): :return None: """ self._last_heartbeat = datetime.now() - logger.info("Heartbeat received from service %r for question %r.", self.child_sruid, self.question_uuid) + logger.info( + "%r: Received a heartbeat from service %r for question %r.", + self.recipient, + self.child_sruid, + self.question_uuid, + ) def _handle_monitor_message(self, event): - """Send a monitor message to the handler if one has been provided. + """Send the monitor message to the handler if one has been provided. :param dict event: :return None: """ - logger.debug("%r received a monitor message.", self.recipient) + logger.debug( + "%r: Received a monitor message from service %r for question %r.", + self.recipient, + self.child_sruid, + self.question_uuid, + ) if self.handle_monitor_message is not None: self.handle_monitor_message(event["data"]) def _handle_log_message(self, event): - """Deserialise the event into a log record and pass it to the local log handlers, adding the child's SRUID to - the start of the log message. + """Deserialise the event into a log record and pass it to the local log handlers. The child's SRUID and the + question UUID are added to the start of the log message, and the SRUIDs of any subchildren called by the child + are each coloured differently. :param dict event: :return None: @@ -291,7 +310,7 @@ def _handle_log_message(self, event): logger.handle(record) def _handle_exception(self, event): - """Raise the exception from the responding service that is serialised in `data`. + """Raise the exception from the child. :param dict event: :raise Exception: @@ -315,12 +334,12 @@ def _handle_exception(self, event): raise exception_type(exception_message) def _handle_result(self, event): - """Convert the result to the correct form, deserialising the output manifest if it is present in the event. + """Extract any output values and output manifest from the result, deserialising the manifest if present. :param dict event: :return dict: """ - logger.info("%r received an answer to question %r.", self.recipient, self.question_uuid) + logger.info("%r: Received an answer to question %r.", self.recipient, self.question_uuid) if event.get("output_manifest"): output_manifest = Manifest.deserialise(event["output_manifest"]) From 43f729295f39b0e9d77011429ab387b1b5cce16e Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 10 Apr 2024 11:11:52 +0100 Subject: [PATCH 133/169] FIX: Handle invalid events correctly --- octue/cloud/events/handler.py | 13 ++++++++++--- octue/cloud/events/validation.py | 3 ++- octue/cloud/pub_sub/service.py | 3 ++- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/octue/cloud/events/handler.py b/octue/cloud/events/handler.py index 90994814e..3b7567421 100644 --- a/octue/cloud/events/handler.py +++ b/octue/cloud/events/handler.py @@ -36,7 +36,7 @@ class AbstractEventHandler: To create a concrete handler for a specific service/communication backend synchronously or asynchronously, inherit from this class and add the `handle_events` and `_extract_event_and_attributes` methods. - :param octue.cloud.pub_sub.service.Service recipient: the service instances that's receiving the events + :param octue.cloud.pub_sub.service.Service recipient: the `Service` instance that's receiving the events :param callable|None handle_monitor_message: a function to handle monitor messages (e.g. send them to an endpoint for plotting or displaying) - this function should take a single JSON-compatible python primitive :param bool record_events: if `True`, record received events in the `received_events` attribute :param dict|None event_handlers: a mapping of event type names to callables that handle each type of event. The handlers must not mutate the events. @@ -130,14 +130,21 @@ def _extract_and_enqueue_event(self, container): :param any container: the container of the event (e.g. a Pub/Sub message) :return None: """ - event, attributes = self._extract_event_and_attributes(container) + try: + event, attributes = self._extract_event_and_attributes(container) + except Exception: + event = None + attributes = {} + + # Don't assume the presence of specific attributes before validation. + child_sdk_version = attributes.get("sender_sdk_version") if not is_event_valid( event=event, attributes=attributes, recipient=self.recipient, parent_sdk_version=PARENT_SDK_VERSION, - child_sdk_version=attributes["sender_sdk_version"], + child_sdk_version=child_sdk_version, schema=self.schema, ): return diff --git a/octue/cloud/events/validation.py b/octue/cloud/events/validation.py index 39dfd003f..d906cf8d4 100644 --- a/octue/cloud/events/validation.py +++ b/octue/cloud/events/validation.py @@ -12,7 +12,7 @@ SERVICE_COMMUNICATION_SCHEMA_INFO_URL = "https://strands.octue.com/octue/service-communication" SERVICE_COMMUNICATION_SCHEMA_VERSION = os.path.splitext(SERVICE_COMMUNICATION_SCHEMA["$ref"])[0].split("/")[-1] -# Instantiate a JSON schema validator to cache the service communication schema. This avoids getting it from the +# Instantiate a JSON schema validator to cache the service communication schema. This avoids downloading it from the # registry every time a message is validated against it. jsonschema.Draft202012Validator.check_schema(SERVICE_COMMUNICATION_SCHEMA) jsonschema_validator = jsonschema.Draft202012Validator(SERVICE_COMMUNICATION_SCHEMA) @@ -56,6 +56,7 @@ def raise_if_event_is_invalid(event, attributes, recipient, parent_sdk_version, :raise jsonschema.ValidationError: if the event or its attributes are invalid :return None: """ + # Transform attributes to a dictionary in the case they're a different kind of mapping. data = {"event": event, "attributes": dict(attributes)} if schema is None: diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index cdbcc443a..b7860631f 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -621,7 +621,8 @@ def _parse_question(self, question): event=event_for_validation, attributes=attributes, recipient=self, - parent_sdk_version=attributes["sender_sdk_version"], + # Don't assume the presence of specific attributes before validation. + parent_sdk_version=attributes.get("sender_sdk_version"), child_sdk_version=importlib.metadata.version("octue"), ) From da845f9da84d02217ed4f18d83de1add622aba49 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 10 Apr 2024 11:13:59 +0100 Subject: [PATCH 134/169] ENH: Make `EventReplayer` clearer and improve logging --- octue/cloud/events/replayer.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/octue/cloud/events/replayer.py b/octue/cloud/events/replayer.py index fb1dec272..22268824d 100644 --- a/octue/cloud/events/replayer.py +++ b/octue/cloud/events/replayer.py @@ -10,14 +10,14 @@ class EventReplayer(AbstractEventHandler): - """A replayer for events retrieved asynchronously from some kind of storage. + """A replayer for events retrieved asynchronously from storage. Missing events are immediately skipped. - :param octue.cloud.pub_sub.service.Service recipient: the service that's receiving the events + :param octue.cloud.pub_sub.service.Service recipient: the `Service` instance that's receiving the events :param callable|None handle_monitor_message: a function to handle monitor messages (e.g. send them to an endpoint for plotting or displaying) - this function should take a single JSON-compatible python primitive :param bool record_events: if `True`, record received events in the `received_events` attribute :param dict|None event_handlers: a mapping of event type names to callables that handle each type of event. The handlers should not mutate the events. :param dict|str schema: the JSON schema (or URI of one) to validate events against - :param bool only_handle_result: if `True`, skip non-result events and only handle the result event + :param bool only_handle_result: if `True`, skip non-result events and only handle the "result" event if present :return None: """ @@ -41,9 +41,10 @@ def __init__( ) def handle_events(self, events): - """Handle the given events and return a handled "result" event if one is reached. + """Handle the given events and return a handled "result" event if one is present. - :return dict: the handled final result + :param iter(dict) events: the events to handle + :return dict|None: the handled "result" event if present """ self.waiting_events = {} self._previous_event_number = -1 @@ -52,9 +53,12 @@ def handle_events(self, events): self._extract_and_enqueue_event(event) # Handle the case where no events (or no valid events) have been received. - if self.waiting_events: - self._earliest_waiting_event_number = min(self.waiting_events.keys()) - return self._attempt_to_handle_waiting_events() + if not self.waiting_events: + logger.warning("No events (or no valid events) were received.") + return + + self._earliest_waiting_event_number = min(self.waiting_events.keys()) + return self._attempt_to_handle_waiting_events() def _extract_event_and_attributes(self, container): """Extract an event and its attributes from the event container. From fc8e3b6ac76864dc35c20c8bc0b95ad72a55401d Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 10 Apr 2024 11:14:26 +0100 Subject: [PATCH 135/169] TST: Test `EventHandler` with no, no valid, or no result events --- tests/cloud/events/test_replayer.py | 44 +++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/tests/cloud/events/test_replayer.py b/tests/cloud/events/test_replayer.py index 7bf95ba65..2cbd43d1a 100644 --- a/tests/cloud/events/test_replayer.py +++ b/tests/cloud/events/test_replayer.py @@ -7,8 +7,48 @@ class TestEventReplayer(unittest.TestCase): - def test_replay_events(self): - """Test that stored events can be replayed and the result extracted.""" + def test_with_no_events(self): + """Test that `None` is returned if no events are passed in.""" + with self.assertLogs() as logging_context: + result = EventReplayer().handle_events(events=[]) + + self.assertIsNone(result) + self.assertIn("No events (or no valid events) were received.", logging_context.output[0]) + + def test_with_no_valid_events(self): + """Test that `None` is returned if no valid events are received.""" + with self.assertLogs() as logging_context: + result = EventReplayer().handle_events(events=[{"invalid": "event"}]) + + self.assertIsNone(result) + self.assertIn("received an event that doesn't conform", logging_context.output[1]) + + def test_no_result_event(self): + """Test that `None` is returned if no result event is received.""" + event = { + "event": { + "datetime": "2024-03-06T15:44:18.156044", + "kind": "delivery_acknowledgement", + }, + "attributes": { + "order": "0", + "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", + "originator": "octue/test-service:1.0.0", + "sender": "octue/test-service:1.0.0", + "sender_type": "CHILD", + "sender_sdk_version": "0.51.0", + "recipient": "octue/another-service:3.2.1", + }, + } + + with self.assertLogs() as logging_context: + result = EventReplayer().handle_events(events=[event]) + + self.assertIsNone(result) + self.assertIn("question was delivered", logging_context.output[0]) + + def test_with_events(self): + """Test that stored events can be replayed and the outputs extracted from the "result" event.""" with open(os.path.join(TESTS_DIR, "data", "events.json")) as f: events = json.load(f) From 0d09f77e67191bdca77c994b091d65c9dac0c866 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 10 Apr 2024 11:30:15 +0100 Subject: [PATCH 136/169] FIX: Simplify Pub/Sub event extraction and avoid pre-validation errors --- octue/cloud/pub_sub/events.py | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/octue/cloud/pub_sub/events.py b/octue/cloud/pub_sub/events.py index 115d8602f..99590b024 100644 --- a/octue/cloud/pub_sub/events.py +++ b/octue/cloud/pub_sub/events.py @@ -1,5 +1,4 @@ import base64 -import importlib.metadata import json import logging import time @@ -18,7 +17,6 @@ logger = logging.getLogger(__name__) MAX_SIMULTANEOUS_MESSAGES_PULL = 50 -PARENT_SDK_VERSION = importlib.metadata.version("octue") def extract_event_and_attributes_from_pub_sub_message(message): @@ -31,25 +29,18 @@ def extract_event_and_attributes_from_pub_sub_message(message): # Cast attributes to a dictionary to avoid defaultdict-like behaviour from Pub/Sub message attributes container. attributes = dict(getattr_or_subscribe(message, "attributes")) - # Required for all events. - converted_attributes = { - "question_uuid": attributes["question_uuid"], - "order": int(attributes["order"]), - "originator": attributes["originator"], - "sender": attributes["sender"], - "sender_type": attributes["sender_type"], - "sender_sdk_version": attributes["sender_sdk_version"], - "recipient": attributes["recipient"], - } + # Deserialise the `order` and `forward_logs` fields if they're present (don't assume they are before validation). + if attributes.get("order"): + attributes["order"] = int(attributes["order"]) # Required for question events. - if attributes["sender_type"] == "PARENT": - converted_attributes.update( - { - "forward_logs": bool(int(attributes["forward_logs"])), - "save_diagnostics": attributes["save_diagnostics"], - } - ) + if attributes.get("sender_type") == "PARENT": + forward_logs = attributes.get("forward_logs") + + if forward_logs: + attributes["forward_logs"] = bool(int(forward_logs)) + else: + attributes["forward_logs"] = None try: # Parse event directly from Pub/Sub or Dataflow. @@ -58,7 +49,7 @@ def extract_event_and_attributes_from_pub_sub_message(message): # Parse event from Google Cloud Run. event = json.loads(base64.b64decode(message["data"]).decode("utf-8").strip(), cls=OctueJSONDecoder) - return event, converted_attributes + return event, attributes class GoogleCloudPubSubEventHandler(AbstractEventHandler): From 664d42cd6c3c892a4b202d09b79da94f90b2fda0 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 10 Apr 2024 11:46:30 +0100 Subject: [PATCH 137/169] ENH: Improve `GoogleCloudPubSubEventHandler` docstrings and logs --- octue/cloud/events/replayer.py | 4 +-- octue/cloud/pub_sub/events.py | 50 ++++++++++++++++------------------ 2 files changed, 26 insertions(+), 28 deletions(-) diff --git a/octue/cloud/events/replayer.py b/octue/cloud/events/replayer.py index 22268824d..6f07ceaf3 100644 --- a/octue/cloud/events/replayer.py +++ b/octue/cloud/events/replayer.py @@ -15,8 +15,8 @@ class EventReplayer(AbstractEventHandler): :param octue.cloud.pub_sub.service.Service recipient: the `Service` instance that's receiving the events :param callable|None handle_monitor_message: a function to handle monitor messages (e.g. send them to an endpoint for plotting or displaying) - this function should take a single JSON-compatible python primitive :param bool record_events: if `True`, record received events in the `received_events` attribute - :param dict|None event_handlers: a mapping of event type names to callables that handle each type of event. The handlers should not mutate the events. - :param dict|str schema: the JSON schema (or URI of one) to validate events against + :param dict|None event_handlers: a mapping of event type names to callables that handle each type of event. The handlers must not mutate the events. + :param dict|str schema: the JSON schema to validate events against :param bool only_handle_result: if `True`, skip non-result events and only handle the "result" event if present :return None: """ diff --git a/octue/cloud/pub_sub/events.py b/octue/cloud/pub_sub/events.py index 99590b024..9f93abe2a 100644 --- a/octue/cloud/pub_sub/events.py +++ b/octue/cloud/pub_sub/events.py @@ -56,11 +56,11 @@ class GoogleCloudPubSubEventHandler(AbstractEventHandler): """A synchronous handler for events received as Google Pub/Sub messages from a pull subscription. :param octue.cloud.pub_sub.subscription.Subscription subscription: the subscription messages are pulled from - :param octue.cloud.pub_sub.service.Service recipient: the service that's receiving the events + :param octue.cloud.pub_sub.service.Service recipient: the `Service` instance that's receiving the events :param callable|None handle_monitor_message: a function to handle monitor messages (e.g. send them to an endpoint for plotting or displaying) - this function should take a single JSON-compatible python primitive :param bool record_events: if `True`, record received events in the `received_events` attribute - :param dict|None event_handlers: a mapping of event type names to callables that handle each type of event. The handlers should not mutate the events. - :param dict|str schema: the JSON schema (or URI of one) to validate events against + :param dict|None event_handlers: a mapping of event type names to callables that handle each type of event. The handlers must not mutate the events. + :param dict|str schema: the JSON schema to validate events against :param int|float skip_missing_events_after: the number of seconds after which to skip any events if they haven't arrived but subsequent events have :return None: """ @@ -94,10 +94,10 @@ def __init__( @property def total_run_time(self): - """Get the amount of time elapsed since `self.handle_events` was called. If it hasn't been called yet, it will - be `None`. + """The amount of time elapsed since `self.handle_events` was called. If it hasn't been called yet, this is + `None`. - :return float|None: the amount of time since `self.handle_events` was called (in seconds) + :return float|None: the amount of time [s] since `self.handle_events` was called """ if self._start_time is None: return None @@ -106,7 +106,7 @@ def total_run_time(self): @property def _time_since_last_heartbeat(self): - """Get the time period since the last heartbeat was received. + """The amount of time since the last heartbeat was received. If no heartbeat has been received, this is `None`. :return datetime.timedelta|None: """ @@ -116,13 +116,13 @@ def _time_since_last_heartbeat(self): return datetime.now() - self._last_heartbeat def handle_events(self, timeout=60, maximum_heartbeat_interval=300): - """Pull events fromthe subscription and handle them in the order they were sent until a "result" event is + """Pull events from the subscription and handle them in the order they were sent until a "result" event is handled, then return the handled result. :param float|None timeout: how long to wait for an answer before raising a `TimeoutError` :param int|float maximum_heartbeat_interval: the maximum amount of time [s] allowed between child heartbeats before an error is raised :raise TimeoutError: if the timeout is exceeded before receiving the final event - :return dict: the handled final result + :return dict: the handled "result" event """ self._start_time = time.perf_counter() self.waiting_events = {} @@ -156,9 +156,9 @@ def handle_events(self, timeout=60, maximum_heartbeat_interval=300): def _monitor_heartbeat(self, maximum_heartbeat_interval): """Change the alive status to `False` and cancel the heartbeat checker if a heartbeat hasn't been received - within the maximum allowed time interval measured from the moment of calling. + within the maximum allowed time interval since the last received heartbeat. - :param float|int maximum_heartbeat_interval: the maximum amount of time (in seconds) allowed between child heartbeats without raising an error + :param float|int maximum_heartbeat_interval: the maximum amount of time [s] allowed between child heartbeats without raising an error :return None: """ maximum_heartbeat_interval = timedelta(seconds=maximum_heartbeat_interval) @@ -171,12 +171,12 @@ def _monitor_heartbeat(self, maximum_heartbeat_interval): self._heartbeat_checker.cancel() def _check_timeout_and_get_pull_timeout(self, timeout): - """Check if the timeout has been exceeded and, if it hasn't, return the timeout for the next message pull. If - the timeout has been exceeded, raise an error. + """Check if the message handling timeout has been exceeded and, if it hasn't, calculate and return the timeout + for the next message pull. If the timeout has been exceeded, raise an error. - :param int|float|None timeout: the timeout for handling all messages + :param int|float|None timeout: the timeout [s] for handling all messages, or `None` if there's no timeout :raise TimeoutError: if the timeout has been exceeded - :return int|float: the timeout for the next message pull in seconds + :return int|float|None: the timeout for the next message pull [s], or `None` if there's no timeout """ if timeout is None: return None @@ -186,9 +186,7 @@ def _check_timeout_and_get_pull_timeout(self, timeout): total_run_time = self.total_run_time if total_run_time > timeout: - raise TimeoutError( - f"No final answer received from topic {self.subscription.topic.path!r} after {timeout} seconds." - ) + raise TimeoutError(f"No final result received from {self.subscription.topic!r} after {timeout} seconds.") return timeout - total_run_time @@ -196,7 +194,7 @@ def _pull_and_enqueue_available_events(self, timeout): """Pull as many events from the subscription as are available and enqueue them in `self.waiting_events`, raising a `TimeoutError` if the timeout is exceeded before succeeding. - :param float|None timeout: how long to wait in seconds for the event before raising a `TimeoutError` + :param float|None timeout: how long to wait for the event [s] before raising a `TimeoutError` :raise TimeoutError|concurrent.futures.TimeoutError: if the timeout is exceeded :return None: """ @@ -220,9 +218,7 @@ def _pull_and_enqueue_available_events(self, timeout): pull_run_time = time.perf_counter() - pull_start_time if timeout is not None and pull_run_time > timeout: - raise TimeoutError( - f"No message received from topic {self.subscription.topic.path!r} after {timeout} seconds.", - ) + raise TimeoutError(f"No message received from {self.subscription.topic!r} after {timeout} seconds.") if not pull_response.received_messages: return @@ -238,13 +234,15 @@ def _pull_and_enqueue_available_events(self, timeout): self._extract_and_enqueue_event(event) # Handle the case where no events (or no valid events) have been received. - if self.waiting_events: - self._earliest_waiting_event_number = min(self.waiting_events.keys()) + if not self.waiting_events: + return + + self._earliest_waiting_event_number = min(self.waiting_events.keys()) def _extract_event_and_attributes(self, container): - """Extract an event and its attributes from the Pub/Sub message. + """Extract an event and its attributes from a Pub/Sub message. - :param dict container: the container of the event + :param dict container: a Pub/Sub message :return (any, dict): the event and its attributes """ return extract_event_and_attributes_from_pub_sub_message(container.message) From 0adf534c2bd68e4a937a8ad1083b28cb91ee6c48 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 10 Apr 2024 12:00:25 +0100 Subject: [PATCH 138/169] ENH: Improve log messages and docstrings in `Service` --- octue/cloud/pub_sub/service.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index b7860631f..30728e007 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -128,7 +128,7 @@ def services_topic(self): topic = Topic(name=OCTUE_SERVICES_PREFIX, project_name=self.backend.project_name) if not topic.exists(): - raise octue.exceptions.ServiceNotFound(f"Topic {topic.name!r} cannot be found.") + raise octue.exceptions.ServiceNotFound(f"{topic!r} cannot be found.") self._services_topic = topic @@ -197,7 +197,7 @@ def serve(self, timeout=None, delete_topic_and_subscription_on_exit=False, allow subscription.delete() except Exception: - logger.error("Deletion of subscription %r failed.", subscription.name) + logger.error("Deletion of %r failed.", subscription) subscriber.close() @@ -423,7 +423,7 @@ def wait_for_answer( def send_exception(self, question_uuid, originator, order, timeout=30): """Serialise and send the exception being handled to the parent. - :param str question_uuid: + :param str question_uuid: the UUID of the question this event relates to :param str originator: the SRUID of the service that asked the question this event is related to :param octue.cloud.events.counter.EventCounter order: an event counter keeping track of the order of emitted events :param float|None timeout: time in seconds to keep retrying sending of the exception @@ -538,7 +538,7 @@ def _send_question( def _send_delivery_acknowledgment(self, question_uuid, originator, order, timeout=30): """Send an acknowledgement of question receipt to the parent. - :param str question_uuid: + :param str question_uuid: the UUID of the question this event relates to :param str originator: the SRUID of the service that asked the question this event is related to :param octue.cloud.events.counter.EventCounter order: an event counter keeping track of the order of emitted events :param float timeout: time in seconds after which to give up sending @@ -561,7 +561,7 @@ def _send_delivery_acknowledgment(self, question_uuid, originator, order, timeou def _send_heartbeat(self, question_uuid, originator, order, timeout=30): """Send a heartbeat to the parent, indicating that the service is alive. - :param str question_uuid: + :param str question_uuid: the UUID of the question this event relates to :param str originator: the SRUID of the service that asked the question this event is related to :param octue.cloud.events.counter.EventCounter order: an event counter keeping track of the order of emitted events :param float timeout: time in seconds after which to give up sending @@ -585,7 +585,7 @@ def _send_monitor_message(self, data, question_uuid, originator, order, timeout= """Send a monitor message to the parent. :param any data: the data to send as a monitor message - :param str question_uuid: + :param str question_uuid: the UUID of the question this event relates to :param str originator: the SRUID of the service that asked the question this event is related to :param octue.cloud.events.counter.EventCounter order: an event counter keeping track of the order of emitted events :param float timeout: time in seconds to retry sending the message From de329a7bd2e89b6196227cba00a2294ae87c8333 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 10 Apr 2024 12:00:53 +0100 Subject: [PATCH 139/169] ENH: Return push subscriptions from `Child.ask` --- octue/resources/child.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/octue/resources/child.py b/octue/resources/child.py index 68a7a86c1..62d970a25 100644 --- a/octue/resources/child.py +++ b/octue/resources/child.py @@ -83,7 +83,7 @@ def ask( :param bool record_messages: if `True`, record messages received from the child in the `received_messages` property :param str save_diagnostics: must be one of {"SAVE_DIAGNOSTICS_OFF", "SAVE_DIAGNOSTICS_ON_CRASH", "SAVE_DIAGNOSTICS_ON"}; if turned on, allow the input values and manifest (and its datasets) to be saved by the child either all the time or just if it fails while processing them :param str|None question_uuid: the UUID to use for the question if a specific one is needed; a UUID is generated if not - :param str|None push_endpoint: if answers to the question should be pushed to an endpoint, provide its URL here; if not, leave this as `None` + :param str|None push_endpoint: if answers to the question should be pushed to an endpoint, provide its URL here (the returned subscription will be a push subscription); if not, leave this as `None` :param bool asynchronous: if `True`, don't create an answer subscription :param float timeout: time in seconds to wait for an answer before raising a timeout error :param float|int maximum_heartbeat_interval: the maximum amount of time (in seconds) allowed between child heartbeats before an error is raised @@ -105,7 +105,7 @@ def ask( ) if push_endpoint or asynchronous: - return None + return subscription return self._service.wait_for_answer( subscription=subscription, From f713d4005c8efb9d2fb76e1cdb848842c190cf04 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 10 Apr 2024 12:03:08 +0100 Subject: [PATCH 140/169] REF: Rename `service_id` to `recipient` --- octue/cloud/pub_sub/service.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index 30728e007..6a6ead92b 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -370,10 +370,10 @@ def ask( input_values=input_values, input_manifest=input_manifest, children=children, - service_id=service_id, forward_logs=subscribe_to_logs, save_diagnostics=save_diagnostics, question_uuid=question_uuid, + recipient=service_id, ) return answer_subscription, question_uuid @@ -493,10 +493,10 @@ def _send_question( input_values, input_manifest, children, - service_id, forward_logs, save_diagnostics, question_uuid, + recipient, timeout=30, ): """Send a question to a child service. @@ -504,10 +504,10 @@ def _send_question( :param any|None input_values: any input values for the question :param octue.resources.manifest.Manifest|None input_manifest: an input manifest of any datasets needed for the question :param list(dict)|None children: a list of children for the child to use instead of its default children (if it uses children). These should be in the same format as in an app's app configuration file and have the same keys. - :param str service_id: the ID of the child to send the question to :param bool forward_logs: whether to request the child to forward its logs :param str save_diagnostics: must be one of {"SAVE_DIAGNOSTICS_OFF", "SAVE_DIAGNOSTICS_ON_CRASH", "SAVE_DIAGNOSTICS_ON"}; if turned on, allow the input values and manifest (and its datasets) to be saved by the child either all the time or just if it fails while processing them :param str question_uuid: the UUID of the question being sent + :param str recipient: the SRUID of the child the question is intended for :param float timeout: time in seconds after which to give up sending :return None: """ @@ -521,7 +521,7 @@ def _send_question( event=question, timeout=timeout, originator=self.id, - recipient=service_id, + recipient=recipient, order=EventCounter(), attributes={ "question_uuid": question_uuid, @@ -533,7 +533,7 @@ def _send_question( # Await successful publishing of the question. future.result() - logger.info("%r asked a question %r to service %r.", self, question_uuid, service_id) + logger.info("%r asked a question %r to service %r.", self, question_uuid, recipient) def _send_delivery_acknowledgment(self, question_uuid, originator, order, timeout=30): """Send an acknowledgement of question receipt to the parent. From a606952989c8b62f00d1c81dadaf48f5ea062b95 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 10 Apr 2024 12:17:52 +0100 Subject: [PATCH 141/169] DOC: Update child docstring --- octue/resources/child.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octue/resources/child.py b/octue/resources/child.py index 62d970a25..ff8d5be83 100644 --- a/octue/resources/child.py +++ b/octue/resources/child.py @@ -88,7 +88,7 @@ def ask( :param float timeout: time in seconds to wait for an answer before raising a timeout error :param float|int maximum_heartbeat_interval: the maximum amount of time (in seconds) allowed between child heartbeats before an error is raised :raise TimeoutError: if the timeout is exceeded while waiting for an answer - :return dict|None: for a synchronous question, a dictionary containing the keys "output_values" and "output_manifest" from the result; for an asynchronous question, `None` + :return dict|octue.cloud.pub_sub.subscription.Subscription|None: for a synchronous question, a dictionary containing the keys "output_values" and "output_manifest" from the result; for a question with a push endpoint, the push subscription; for an asynchronous question, `None` """ subscription, _ = self._service.ask( service_id=self.id, From 7070f01fdf64ab14f381ed46f4d6b59526f9d757 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 10 Apr 2024 12:18:04 +0100 Subject: [PATCH 142/169] TST: Restore unmodified service tests --- tests/cloud/pub_sub/test_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/cloud/pub_sub/test_service.py b/tests/cloud/pub_sub/test_service.py index a03ce9ec4..d192b2c71 100644 --- a/tests/cloud/pub_sub/test_service.py +++ b/tests/cloud/pub_sub/test_service.py @@ -181,7 +181,7 @@ def test_timeout_error_raised_if_no_messages_received_when_waiting(self): with self.assertRaises(TimeoutError): service.wait_for_answer(subscription=mock_subscription, timeout=0.01) - def test_error_raised_if_attempting_to_wait_for_answer_from_non_pull_subscription(self): + def test_error_raised_if_attempting_to_wait_for_answer_from_push_subscription(self): """Test that an error is raised if attempting to wait for an answer from a push subscription.""" service = Service(backend=BACKEND) @@ -324,7 +324,7 @@ def mock_app(analysis): with self.assertLogs(level=logging.ERROR) as logs_context_manager: child.serve() subscription, _ = parent.ask(service_id=child.id, input_values={}, subscribe_to_logs=True) - parent.wait_for_answer(subscription, timeout=100) + parent.wait_for_answer(subscription) error_logged = False From 273dcd9b66c48db872a52d286ccdfcdfab9d7167 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 10 Apr 2024 12:20:56 +0100 Subject: [PATCH 143/169] TST: Add assertion to test --- tests/cloud/events/test_replayer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/cloud/events/test_replayer.py b/tests/cloud/events/test_replayer.py index 2cbd43d1a..3bfa8783e 100644 --- a/tests/cloud/events/test_replayer.py +++ b/tests/cloud/events/test_replayer.py @@ -22,6 +22,7 @@ def test_with_no_valid_events(self): self.assertIsNone(result) self.assertIn("received an event that doesn't conform", logging_context.output[1]) + self.assertIn("No events (or no valid events) were received.", logging_context.output[2]) def test_no_result_event(self): """Test that `None` is returned if no result event is received.""" From 2e9547ac663b978362152ae248b57a5bc4034a62 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 10 Apr 2024 14:40:48 +0100 Subject: [PATCH 144/169] TST: Improve `GoogleCloudPubSubEventHandler` tests --- tests/cloud/pub_sub/test_events.py | 55 +++++++++++++++++++++--------- 1 file changed, 39 insertions(+), 16 deletions(-) diff --git a/tests/cloud/pub_sub/test_events.py b/tests/cloud/pub_sub/test_events.py index 749e0bf02..9168d3cc0 100644 --- a/tests/cloud/pub_sub/test_events.py +++ b/tests/cloud/pub_sub/test_events.py @@ -12,7 +12,7 @@ from tests.base import BaseTestCase -class TestPubSubEventHandler(BaseTestCase): +class TestGoogleCloudPubSubEventHandler(BaseTestCase): service_patcher = ServicePatcher() @classmethod @@ -24,7 +24,7 @@ def setUpClass(cls): cls.topic.create(allow_existing=True) cls.subscription = MockSubscription( - name=f"my-org.my-service.1-0-0.answers.{cls.question_uuid}", + name=f"octue.services.my-org.my-service.1-0-0.answers.{cls.question_uuid}", topic=cls.topic, ) cls.subscription.create() @@ -405,7 +405,14 @@ def test_missing_messages_at_start_can_be_skipped(self): order=message["event"]["order"], ) - result = event_handler.handle_events() + with self.assertLogs() as logging_context: + result = event_handler.handle_events() + + self.assertIn( + f"2 consecutive events missing for question {self.question_uuid!r} after 0s - skipping to next earliest " + f"waiting event (event 2).", + logging_context.output[0], + ) self.assertEqual(result, "This is the result.") self.assertEqual( @@ -465,7 +472,14 @@ def test_missing_messages_in_middle_can_skipped(self): order=5, ) - event_handler.handle_events() + with self.assertLogs() as logging_context: + event_handler.handle_events() + + self.assertIn( + f"2 consecutive events missing for question {self.question_uuid!r} after 0s - skipping to next earliest " + f"waiting event (event 5).", + logging_context.output[0], + ) # Check that all the non-missing messages were handled. self.assertEqual( @@ -555,7 +569,20 @@ def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): order=message["event"]["order"], ) - event_handler.handle_events() + with self.assertLogs() as logging_context: + event_handler.handle_events() + + self.assertIn( + f"2 consecutive events missing for question {self.question_uuid!r} after 0s - skipping to next earliest " + f"waiting event (event 5).", + logging_context.output[0], + ) + + self.assertIn( + f"14 consecutive events missing for question {self.question_uuid!r} after 0s - skipping to next earliest " + f"waiting event (event 20).", + logging_context.output[1], + ) # Check that all the non-missing messages were handled. self.assertEqual( @@ -594,8 +621,14 @@ def test_all_messages_missing_apart_from_result(self): order=1000, ) - event_handler.handle_events() + with self.assertLogs() as logging_context: + event_handler.handle_events() + self.assertIn( + f"1000 consecutive events missing for question {self.question_uuid!r} after 0s - skipping to next earliest " + f"waiting event (event 1000).", + logging_context.output[0], + ) # Check that the result message was handled. self.assertEqual(event_handler.handled_events, [{"kind": "finish-test", "order": 1000}]) @@ -632,11 +665,6 @@ def tearDownClass(cls): def test_pull_and_enqueue_available_events(self): """Test that pulling and enqueuing a message works.""" - self.subscription = MockSubscription( - name=f"my-org.my-service.1-0-0.answers.{self.question_uuid}", - topic=self.topic, - ) - event_handler = GoogleCloudPubSubEventHandler( subscription=self.subscription, recipient=self.parent, @@ -673,11 +701,6 @@ def test_pull_and_enqueue_available_events(self): def test_timeout_error_raised_if_result_message_not_received_in_time(self): """Test that a timeout error is raised if a result message is not received in time.""" - self.subscription = MockSubscription( - name=f"my-org.my-service.1-0-0.answers.{self.question_uuid}", - topic=self.topic, - ) - event_handler = GoogleCloudPubSubEventHandler( subscription=self.subscription, recipient=self.parent, From ad8e26a5afaea6c3ab7711a882c3ca26f22f6f95 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 10 Apr 2024 15:43:22 +0100 Subject: [PATCH 145/169] OPS: Update bigquery table and cloud function --- terraform/bigquery.tf | 14 +++----------- terraform/functions.tf | 2 +- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/terraform/bigquery.tf b/terraform/bigquery.tf index a4cb12b98..4698ed662 100644 --- a/terraform/bigquery.tf +++ b/terraform/bigquery.tf @@ -10,11 +10,8 @@ resource "google_bigquery_dataset" "test_dataset" { resource "google_bigquery_table" "test_table" { dataset_id = google_bigquery_dataset.test_dataset.dataset_id - table_id = "question-events" - - labels = { - env = "default" - } + table_id = "service-events" + clustering = ["sender", "question_uuid"] schema = < Date: Wed, 10 Apr 2024 16:29:20 +0100 Subject: [PATCH 146/169] ENH: Update `get_events` function for new event store schema skipci --- octue/cloud/events/validation.py | 4 +- octue/cloud/pub_sub/bigquery.py | 65 +++++++++++++++++++--------- tests/cloud/pub_sub/test_bigquery.py | 50 ++++++++++++++++----- 3 files changed, 85 insertions(+), 34 deletions(-) diff --git a/octue/cloud/events/validation.py b/octue/cloud/events/validation.py index d906cf8d4..c79594965 100644 --- a/octue/cloud/events/validation.py +++ b/octue/cloud/events/validation.py @@ -6,7 +6,7 @@ from octue.compatibility import warn_if_incompatible -logger = logging.getLogger(__name__) +VALID_EVENT_KINDS = {"delivery_acknowledgement", "heartbeat", "log_record", "monitor_message", "exception", "result"} SERVICE_COMMUNICATION_SCHEMA = {"$ref": "https://jsonschema.registry.octue.com/octue/service-communication/0.9.0.json"} SERVICE_COMMUNICATION_SCHEMA_INFO_URL = "https://strands.octue.com/octue/service-communication" @@ -17,6 +17,8 @@ jsonschema.Draft202012Validator.check_schema(SERVICE_COMMUNICATION_SCHEMA) jsonschema_validator = jsonschema.Draft202012Validator(SERVICE_COMMUNICATION_SCHEMA) +logger = logging.getLogger(__name__) + def is_event_valid(event, attributes, recipient, parent_sdk_version, child_sdk_version, schema=None): """Check if the event and its attributes are valid according to the Octue services communication schema. diff --git a/octue/cloud/pub_sub/bigquery.py b/octue/cloud/pub_sub/bigquery.py index 409372039..ccb2d9beb 100644 --- a/octue/cloud/pub_sub/bigquery.py +++ b/octue/cloud/pub_sub/bigquery.py @@ -2,46 +2,70 @@ from google.cloud.bigquery import Client, QueryJobConfig, ScalarQueryParameter +from octue.cloud.events.validation import VALID_EVENT_KINDS -VALID_EVENT_KINDS = {"delivery_acknowledgement", "heartbeat", "log_record", "monitor_message", "exception", "result"} - -def get_events(table_id, question_uuid, kind=None, limit=1000, include_pub_sub_metadata=False): - """Get Octue service events for a question from a Google BigQuery table. +def get_events( + table_id, + sender, + question_uuid, + kind=None, + include_attributes=False, + include_backend_metadata=False, + limit=1000, +): + """Get Octue service events for a question from a sender from a Google BigQuery event store. :param str table_id: the full ID of the table e.g. "your-project.your-dataset.your-table" + :param str sender: the SRUID of the sender of the events :param str question_uuid: the UUID of the question to get the events for :param str|None kind: the kind of event to get; if `None`, all event kinds are returned + :param bool include_attributes: if `True`, include events' attributes (excluding question UUID) + :param bool include_backend_metadata: if `True`, include the service backend metadata :param int limit: the maximum number of events to return - :param bool include_pub_sub_metadata: if `True`, include Pub/Sub metadata :return list(dict): the events for the question """ if kind: if kind not in VALID_EVENT_KINDS: raise ValueError(f"`kind` must be one of {VALID_EVENT_KINDS!r}; received {kind!r}.") - event_kind_condition = [f'AND JSON_EXTRACT_SCALAR(data, "$.kind") = "{kind}"'] + event_kind_condition = [f'AND JSON_EXTRACT_SCALAR(event, "$.kind") = "{kind}"'] else: event_kind_condition = [] client = Client() - fields = ["data", "attributes"] + fields = ["`event`"] + + if include_attributes: + fields.extend( + ( + "`originator`", + "`sender`", + "`sender_type`", + "`sender_sdk_version`", + "`recipient`", + "`order`", + "`other_attributes`", + ) + ) - if include_pub_sub_metadata: - fields.extend(("subscription_name", "message_id", "publish_time")) + if include_backend_metadata: + fields.extend(("`backend`", "`backend_metadata`")) query = "\n".join( [ f"SELECT {', '.join(fields)} FROM `{table_id}`", - "WHERE CONTAINS_SUBSTR(subscription_name, @question_uuid)", + "WHERE sender=@sender", + "AND question_uuid=@question_uuid", *event_kind_condition, - "ORDER BY publish_time", + "ORDER BY `order`", "LIMIT @limit", ] ) job_config = QueryJobConfig( query_parameters=[ + ScalarQueryParameter("sender", "STRING", sender), ScalarQueryParameter("question_uuid", "STRING", question_uuid), ScalarQueryParameter("limit", "INTEGER", limit), ] @@ -49,16 +73,15 @@ def get_events(table_id, question_uuid, kind=None, limit=1000, include_pub_sub_m query_job = client.query(query, job_config=job_config) rows = query_job.result() - messages = rows.to_dataframe() + df = rows.to_dataframe() + + # Convert JSON strings to python primitives. + df["event"] = df["event"].map(json.loads) - # Convert JSON to python primitives. - if isinstance(messages.at[0, "data"], str): - messages["data"] = messages["data"].map(json.loads) + if "other_attributes" in df: + df["other_attributes"] = df["other_attributes"].map(json.loads) - if isinstance(messages.at[0, "attributes"], str): - messages["attributes"] = messages["attributes"].map(json.loads) + if "backend_metadata" in df: + df["backend_metadata"] = df["backend_metadata"].map(json.loads) - # Order messages. - messages = messages.iloc[messages["attributes"].str.get("order").astype(str).argsort()] - messages.rename(columns={"data": "event"}, inplace=True) - return messages.to_dict(orient="records") + return df.to_dict(orient="records") diff --git a/tests/cloud/pub_sub/test_bigquery.py b/tests/cloud/pub_sub/test_bigquery.py index 9e9a21eb8..c34e6a403 100644 --- a/tests/cloud/pub_sub/test_bigquery.py +++ b/tests/cloud/pub_sub/test_bigquery.py @@ -8,37 +8,63 @@ class TestBigQuery(TestCase): def test_error_raised_if_event_kind_invalid(self): """Test that an error is raised if the event kind is invalid.""" with self.assertRaises(ValueError): - get_events(table_id="blah", question_uuid="blah", kind="frisbee_tournament") + get_events( + table_id="blah", + sender="octue/test-service:1.0.0", + question_uuid="blah", + kind="frisbee_tournament", + ) def test_without_kind(self): """Test the query used to retrieve events of all kinds.""" with patch("octue.cloud.pub_sub.bigquery.Client") as mock_client: - get_events(table_id="blah", question_uuid="blah") + get_events(table_id="blah", sender="octue/test-service:1.0.0", question_uuid="blah") self.assertEqual( mock_client.mock_calls[1].args[0], - "SELECT data, attributes FROM `blah`\nWHERE CONTAINS_SUBSTR(subscription_name, @question_uuid)\n" - "ORDER BY publish_time\nLIMIT @limit", + "SELECT `event` FROM `blah`\nWHERE sender=@sender\nAND question_uuid=@question_uuid\n" + "ORDER BY `order`\nLIMIT @limit", ) def test_with_kind(self): """Test the query used to retrieve events of a specific kind.""" with patch("octue.cloud.pub_sub.bigquery.Client") as mock_client: - get_events(table_id="blah", question_uuid="blah", kind="result") + get_events(table_id="blah", sender="octue/test-service:1.0.0", question_uuid="blah", kind="result") self.assertEqual( mock_client.mock_calls[1].args[0], - "SELECT data, attributes FROM `blah`\nWHERE CONTAINS_SUBSTR(subscription_name, @question_uuid)\n" - 'AND JSON_EXTRACT_SCALAR(data, "$.kind") = "result"\nORDER BY publish_time\nLIMIT @limit', + "SELECT `event` FROM `blah`\nWHERE sender=@sender\nAND question_uuid=@question_uuid\n" + 'AND JSON_EXTRACT_SCALAR(event, "$.kind") = "result"\nORDER BY `order`\nLIMIT @limit', ) - def test_with_pub_sub_metadata(self): - """Test the query used to retrieve Pub/Sub metadata in addition to events.""" + def test_with_attributes(self): + """Test the query used to retrieve attributes in addition to events.""" with patch("octue.cloud.pub_sub.bigquery.Client") as mock_client: - get_events(table_id="blah", question_uuid="blah", include_pub_sub_metadata=True) + get_events( + table_id="blah", + sender="octue/test-service:1.0.0", + question_uuid="blah", + include_attributes=True, + ) self.assertEqual( mock_client.mock_calls[1].args[0], - "SELECT data, attributes, subscription_name, message_id, publish_time FROM `blah`\n" - "WHERE CONTAINS_SUBSTR(subscription_name, @question_uuid)\nORDER BY publish_time\nLIMIT @limit", + "SELECT `event`, `originator`, `sender`, `sender_type`, `sender_sdk_version`, `recipient`, `order`, `other_attributes` FROM `blah`\n" + "WHERE sender=@sender\nAND question_uuid=@question_uuid\nORDER BY `order`\nLIMIT @limit", + ) + + def test_with_backend_metadata(self): + """Test the query used to retrieve backend metadata in addition to events.""" + with patch("octue.cloud.pub_sub.bigquery.Client") as mock_client: + get_events( + table_id="blah", + sender="octue/test-service:1.0.0", + question_uuid="blah", + include_backend_metadata=True, + ) + + self.assertEqual( + mock_client.mock_calls[1].args[0], + "SELECT `event`, `backend`, `backend_metadata` FROM `blah`\n" + "WHERE sender=@sender\nAND question_uuid=@question_uuid\nORDER BY `order`\nLIMIT @limit", ) From ca610aa1d07558654efd159de89eaf4f9d30bf14 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 10 Apr 2024 17:30:15 +0100 Subject: [PATCH 147/169] OPS: Use fixed event handler cloud function --- terraform/functions.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/functions.tf b/terraform/functions.tf index 1feed4d27..8f77864e3 100644 --- a/terraform/functions.tf +++ b/terraform/functions.tf @@ -9,7 +9,7 @@ resource "google_cloudfunctions2_function" "event_handler" { source { storage_source { bucket = "twined-gcp" - object = "event_handler/0.3.0.zip" + object = "event_handler/0.3.1.zip" } } } From ad39bda35fca988cabf11b397d76a72be750c076 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 10 Apr 2024 17:35:35 +0100 Subject: [PATCH 148/169] REF: Rename `received/record_messages` to `received/record_events` BREAKING CHANGE: Replace the `record_messages` parameter with `record_events` and the `received_messages` property with `received_events`. --- octue/cloud/emulators/child.py | 12 +++++----- octue/cloud/pub_sub/service.py | 14 +++++------ octue/resources/child.py | 12 +++++----- octue/runner.py | 4 +--- .../cloud_run/test_cloud_run_deployment.py | 2 +- tests/cloud/emulators/test_child_emulator.py | 2 +- tests/cloud/pub_sub/test_service.py | 24 +++++++++---------- 7 files changed, 34 insertions(+), 36 deletions(-) diff --git a/octue/cloud/emulators/child.py b/octue/cloud/emulators/child.py index a843d55ee..02d33d409 100644 --- a/octue/cloud/emulators/child.py +++ b/octue/cloud/emulators/child.py @@ -77,12 +77,12 @@ def __repr__(self): return f"<{type(self).__name__}({self.id!r})>" @property - def received_messages(self): - """Get the messages received from the child. + def received_events(self): + """Get the events received from the child. :return list(dict): """ - return self._parent.received_messages + return self._parent.received_events def ask( self, @@ -91,7 +91,7 @@ def ask( subscribe_to_logs=True, allow_local_files=False, handle_monitor_message=None, - record_messages=True, + record_events=True, question_uuid=None, push_endpoint=None, asynchronous=False, @@ -106,7 +106,7 @@ def ask( :param bool subscribe_to_logs: if `True`, subscribe to logs from the child and handle them with the local log handlers :param bool allow_local_files: if `True`, allow the input manifest to contain references to local files - this should only be set to `True` if the child will have access to these local files :param callable|None handle_monitor_message: a function to handle monitor messages (e.g. send them to an endpoint for plotting or displaying) - this function should take a single JSON-compatible python primitive as an argument (note that this could be an array or object) - :param bool record_messages: if `True`, record messages received from the child in the `received_messages` property + :param bool record_events: if `True`, record events received from the child in the `received_events` property :param str|None question_uuid: the UUID to use for the question if a specific one is needed; a UUID is generated if not :param str|None push_endpoint: if answers to the question should be pushed to an endpoint, provide its URL here (the returned subscription will be a push subscription); if not, leave this as `None` :param bool asynchronous: if `True`, don't create an answer subscription @@ -131,7 +131,7 @@ def ask( return self._parent.wait_for_answer( subscription, handle_monitor_message=handle_monitor_message, - record_messages=record_messages, + record_events=record_events, timeout=timeout, ) diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index 6a6ead92b..5abd17ac4 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -135,10 +135,10 @@ def services_topic(self): return self._services_topic @property - def received_messages(self): - """Get the messages received by the service from a child service while running the `wait_for_answer` method. If - the `wait_for_answer` method hasn't been run, `None` is returned. If an empty list is returned, no messages have - been received. + def received_events(self): + """Get the events received from a child service while running the `wait_for_answer` method. If the + `wait_for_answer` method hasn't been run, `None` is returned. If an empty list is returned, no events have been + received. :return list(dict)|None: """ @@ -382,7 +382,7 @@ def wait_for_answer( self, subscription, handle_monitor_message=None, - record_messages=True, + record_events=True, timeout=60, maximum_heartbeat_interval=300, ): @@ -391,7 +391,7 @@ def wait_for_answer( :param octue.cloud.pub_sub.subscription.Subscription subscription: the subscription for the question's answer :param callable|None handle_monitor_message: a function to handle monitor messages (e.g. send them to an endpoint for plotting or displaying) - this function should take a single JSON-compatible python primitive as an argument (note that this could be an array or object) - :param bool record_messages: if `True`, record messages received from the child in the `received_messages` attribute + :param bool record_events: if `True`, record messages received from the child in the `received_events` attribute :param float|None timeout: how long in seconds to wait for an answer before raising a `TimeoutError` :param float|int delivery_acknowledgement_timeout: how long in seconds to wait for a delivery acknowledgement before aborting :param float|int maximum_heartbeat_interval: the maximum amount of time (in seconds) allowed between child heartbeats before an error is raised @@ -408,7 +408,7 @@ def wait_for_answer( subscription=subscription, recipient=self, handle_monitor_message=handle_monitor_message, - record_events=record_messages, + record_events=record_events, ) try: diff --git a/octue/resources/child.py b/octue/resources/child.py index ff8d5be83..80d6377e6 100644 --- a/octue/resources/child.py +++ b/octue/resources/child.py @@ -44,13 +44,13 @@ def __repr__(self): return f"<{type(self).__name__}({self.id!r})>" @property - def received_messages(self): - """Get the messages received from the child if it has been asked a question. If it hasn't, `None` is returned. + def received_events(self): + """Get the events received from the child if it has been asked a question. If it hasn't, `None` is returned. If an empty list is returned, no messages have been received. :return list(dict)|None: """ - return self._service.received_messages + return self._service.received_events def ask( self, @@ -60,7 +60,7 @@ def ask( subscribe_to_logs=True, allow_local_files=False, handle_monitor_message=None, - record_messages=True, + record_events=True, save_diagnostics="SAVE_DIAGNOSTICS_ON_CRASH", # This is repeated as a string here to avoid a circular import. question_uuid=None, push_endpoint=None, @@ -80,7 +80,7 @@ def ask( :param bool subscribe_to_logs: if `True`, subscribe to logs from the child and handle them with the local log handlers :param bool allow_local_files: if `True`, allow the input manifest to contain references to local files - this should only be set to `True` if the child will have access to these local files :param callable|None handle_monitor_message: a function to handle monitor messages (e.g. send them to an endpoint for plotting or displaying) - this function should take a single JSON-compatible python primitive as an argument (note that this could be an array or object) - :param bool record_messages: if `True`, record messages received from the child in the `received_messages` property + :param bool record_events: if `True`, record messages received from the child in the `received_events` property :param str save_diagnostics: must be one of {"SAVE_DIAGNOSTICS_OFF", "SAVE_DIAGNOSTICS_ON_CRASH", "SAVE_DIAGNOSTICS_ON"}; if turned on, allow the input values and manifest (and its datasets) to be saved by the child either all the time or just if it fails while processing them :param str|None question_uuid: the UUID to use for the question if a specific one is needed; a UUID is generated if not :param str|None push_endpoint: if answers to the question should be pushed to an endpoint, provide its URL here (the returned subscription will be a push subscription); if not, leave this as `None` @@ -110,7 +110,7 @@ def ask( return self._service.wait_for_answer( subscription=subscription, handle_monitor_message=handle_monitor_message, - record_messages=record_messages, + record_events=record_events, timeout=timeout, maximum_heartbeat_interval=maximum_heartbeat_interval, ) diff --git a/octue/runner.py b/octue/runner.py index d1fac55d3..c65d6e815 100644 --- a/octue/runner.py +++ b/octue/runner.py @@ -374,9 +374,7 @@ def wrapper(*args, **kwargs): try: return original_ask_method(**kwargs) finally: - self.diagnostics.add_question( - {"id": child.id, "key": key, **kwargs, "messages": child.received_messages} - ) + self.diagnostics.add_question({"id": child.id, "key": key, **kwargs, "messages": child.received_events}) return wrapper diff --git a/tests/cloud/deployment/google/cloud_run/test_cloud_run_deployment.py b/tests/cloud/deployment/google/cloud_run/test_cloud_run_deployment.py index bb6e12903..1169e5f52 100644 --- a/tests/cloud/deployment/google/cloud_run/test_cloud_run_deployment.py +++ b/tests/cloud/deployment/google/cloud_run/test_cloud_run_deployment.py @@ -13,7 +13,7 @@ class TestCloudRunDeployment(TestCase): # This is the service ID of the example service deployed to Google Cloud Run. child = Child( - id="octue/example-service-cloud-run:0.4.0", + id="octue/example-service-cloud-run:stream-event", backend={"name": "GCPPubSubBackend", "project_name": os.environ["TEST_PROJECT_NAME"]}, ) diff --git a/tests/cloud/emulators/test_child_emulator.py b/tests/cloud/emulators/test_child_emulator.py index 4c0af6a00..55cf5bea2 100644 --- a/tests/cloud/emulators/test_child_emulator.py +++ b/tests/cloud/emulators/test_child_emulator.py @@ -280,7 +280,7 @@ def error_run_function(*args, **kwargs): subscription, _ = parent.ask(service_id=child.id, input_values={}) parent.wait_for_answer(subscription=subscription) - child_emulator = ChildEmulator(messages=parent.received_messages) + child_emulator = ChildEmulator(messages=parent.received_events) with self.assertRaises(OSError): child_emulator.ask(input_values={}) diff --git a/tests/cloud/pub_sub/test_service.py b/tests/cloud/pub_sub/test_service.py index d192b2c71..c0b10889c 100644 --- a/tests/cloud/pub_sub/test_service.py +++ b/tests/cloud/pub_sub/test_service.py @@ -699,11 +699,11 @@ def test_child_messages_can_be_recorded_by_parent(self): parent.wait_for_answer(subscription) # Check that the child's messages have been recorded by the parent. - self.assertEqual(parent.received_messages[0]["kind"], "delivery_acknowledgement") - self.assertEqual(parent.received_messages[1]["kind"], "log_record") - self.assertEqual(parent.received_messages[2]["kind"], "log_record") - self.assertEqual(parent.received_messages[3]["kind"], "log_record") - self.assertEqual(parent.received_messages[4], {"kind": "result", "output_values": "Hello! It worked!"}) + self.assertEqual(parent.received_events[0]["kind"], "delivery_acknowledgement") + self.assertEqual(parent.received_events[1]["kind"], "log_record") + self.assertEqual(parent.received_events[2]["kind"], "log_record") + self.assertEqual(parent.received_events[3]["kind"], "log_record") + self.assertEqual(parent.received_events[4], {"kind": "result", "output_values": "Hello! It worked!"}) def test_child_exception_message_can_be_recorded_by_parent(self): """Test that the parent can record exceptions raised by the child.""" @@ -716,9 +716,9 @@ def test_child_exception_message_can_be_recorded_by_parent(self): parent.wait_for_answer(subscription) # Check that the child's messages have been recorded by the parent. - self.assertEqual(parent.received_messages[0]["kind"], "delivery_acknowledgement") - self.assertEqual(parent.received_messages[1]["kind"], "exception") - self.assertIn("Oh no.", parent.received_messages[1]["exception_message"]) + self.assertEqual(parent.received_events[0]["kind"], "delivery_acknowledgement") + self.assertEqual(parent.received_events[1]["kind"], "exception") + self.assertIn("Oh no.", parent.received_events[1]["exception_message"]) def test_child_sends_heartbeat_messages_at_expected_regular_intervals(self): """Test that children send heartbeat messages at the expected regular intervals.""" @@ -745,11 +745,11 @@ def run_function(*args, **kwargs): parent.wait_for_answer(subscription) - self.assertEqual(parent.received_messages[1]["kind"], "heartbeat") - self.assertEqual(parent.received_messages[2]["kind"], "heartbeat") + self.assertEqual(parent.received_events[1]["kind"], "heartbeat") + self.assertEqual(parent.received_events[2]["kind"], "heartbeat") - first_heartbeat_time = datetime.datetime.fromisoformat(parent.received_messages[1]["datetime"]) - second_heartbeat_time = datetime.datetime.fromisoformat(parent.received_messages[2]["datetime"]) + first_heartbeat_time = datetime.datetime.fromisoformat(parent.received_events[1]["datetime"]) + second_heartbeat_time = datetime.datetime.fromisoformat(parent.received_events[2]["datetime"]) self.assertAlmostEqual( second_heartbeat_time - first_heartbeat_time, From 64b888ae41ab45eae5328fcd165e8aeb0ecd58e0 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 10 Apr 2024 17:44:37 +0100 Subject: [PATCH 149/169] REF: Factor out resetting event handlers --- octue/cloud/events/handler.py | 14 ++++++++++++-- octue/cloud/events/replayer.py | 3 +-- octue/cloud/pub_sub/events.py | 5 +---- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/octue/cloud/events/handler.py b/octue/cloud/events/handler.py index 3b7567421..2d4c2da74 100644 --- a/octue/cloud/events/handler.py +++ b/octue/cloud/events/handler.py @@ -70,6 +70,7 @@ def __init__( self.waiting_events = None self.handled_events = [] self._previous_event_number = -1 + self._start_time = None self.skip_missing_events_after = skip_missing_events_after self._missing_event_detection_time = None @@ -109,11 +110,20 @@ def time_since_missing_event(self): @abc.abstractmethod def handle_events(self, *args, **kwargs): """Handle events and return a handled "result" event once one is received. This method must be overridden but - can have any arguments. + can have any arguments. The first thing it should do is call `super().handle_events()`. :return dict: the handled final result """ - pass + self.reset() + + def reset(self): + """Reset the handler to be ready to handle a new stream of events. + + :return None: + """ + self._start_time = time.perf_counter() + self.waiting_events = {} + self._previous_event_number = -1 @abc.abstractmethod def _extract_event_and_attributes(self, container): diff --git a/octue/cloud/events/replayer.py b/octue/cloud/events/replayer.py index 6f07ceaf3..082e1d00a 100644 --- a/octue/cloud/events/replayer.py +++ b/octue/cloud/events/replayer.py @@ -46,8 +46,7 @@ def handle_events(self, events): :param iter(dict) events: the events to handle :return dict|None: the handled "result" event if present """ - self.waiting_events = {} - self._previous_event_number = -1 + super().handle_events() for event in events: self._extract_and_enqueue_event(event) diff --git a/octue/cloud/pub_sub/events.py b/octue/cloud/pub_sub/events.py index 9f93abe2a..fbc16fb74 100644 --- a/octue/cloud/pub_sub/events.py +++ b/octue/cloud/pub_sub/events.py @@ -90,7 +90,6 @@ def __init__( self._heartbeat_checker = None self._last_heartbeat = None self._alive = True - self._start_time = None @property def total_run_time(self): @@ -124,9 +123,7 @@ def handle_events(self, timeout=60, maximum_heartbeat_interval=300): :raise TimeoutError: if the timeout is exceeded before receiving the final event :return dict: the handled "result" event """ - self._start_time = time.perf_counter() - self.waiting_events = {} - self._previous_event_number = -1 + super().handle_events() self._heartbeat_checker = RepeatingTimer( interval=maximum_heartbeat_interval, From 9ec3f1f33c5ad16e01cc12f1d9cc0f00b8519f2e Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 10 Apr 2024 17:58:30 +0100 Subject: [PATCH 150/169] REF: Move earliest waiting event number calculation into superclass --- octue/cloud/events/handler.py | 7 +++++++ octue/cloud/events/replayer.py | 6 ------ octue/cloud/pub_sub/events.py | 6 ------ tests/cloud/events/test_replayer.py | 5 +++-- tests/cloud/pub_sub/test_events.py | 1 - 5 files changed, 10 insertions(+), 15 deletions(-) diff --git a/octue/cloud/events/handler.py b/octue/cloud/events/handler.py index 2d4c2da74..1eff9fc9c 100644 --- a/octue/cloud/events/handler.py +++ b/octue/cloud/events/handler.py @@ -186,6 +186,13 @@ def _attempt_to_handle_waiting_events(self): :return any|None: either a handled non-`None` "result" event, or `None` if nothing was returned by the event handlers or if the next in-order event hasn't been received yet """ + # Handle the case where no events (or no valid events) have been received. + if not self.waiting_events: + logger.debug("No events (or no valid events) were received.") + return + + self._earliest_waiting_event_number = min(self.waiting_events.keys()) + while self.waiting_events: try: # If the next consecutive event has been received: diff --git a/octue/cloud/events/replayer.py b/octue/cloud/events/replayer.py index 082e1d00a..203922b46 100644 --- a/octue/cloud/events/replayer.py +++ b/octue/cloud/events/replayer.py @@ -51,12 +51,6 @@ def handle_events(self, events): for event in events: self._extract_and_enqueue_event(event) - # Handle the case where no events (or no valid events) have been received. - if not self.waiting_events: - logger.warning("No events (or no valid events) were received.") - return - - self._earliest_waiting_event_number = min(self.waiting_events.keys()) return self._attempt_to_handle_waiting_events() def _extract_event_and_attributes(self, container): diff --git a/octue/cloud/pub_sub/events.py b/octue/cloud/pub_sub/events.py index fbc16fb74..9d0b700b8 100644 --- a/octue/cloud/pub_sub/events.py +++ b/octue/cloud/pub_sub/events.py @@ -230,12 +230,6 @@ def _pull_and_enqueue_available_events(self, timeout): for event in pull_response.received_messages: self._extract_and_enqueue_event(event) - # Handle the case where no events (or no valid events) have been received. - if not self.waiting_events: - return - - self._earliest_waiting_event_number = min(self.waiting_events.keys()) - def _extract_event_and_attributes(self, container): """Extract an event and its attributes from a Pub/Sub message. diff --git a/tests/cloud/events/test_replayer.py b/tests/cloud/events/test_replayer.py index 3bfa8783e..368ca8a99 100644 --- a/tests/cloud/events/test_replayer.py +++ b/tests/cloud/events/test_replayer.py @@ -1,4 +1,5 @@ import json +import logging import os import unittest @@ -9,7 +10,7 @@ class TestEventReplayer(unittest.TestCase): def test_with_no_events(self): """Test that `None` is returned if no events are passed in.""" - with self.assertLogs() as logging_context: + with self.assertLogs(level=logging.DEBUG) as logging_context: result = EventReplayer().handle_events(events=[]) self.assertIsNone(result) @@ -17,7 +18,7 @@ def test_with_no_events(self): def test_with_no_valid_events(self): """Test that `None` is returned if no valid events are received.""" - with self.assertLogs() as logging_context: + with self.assertLogs(level=logging.DEBUG) as logging_context: result = EventReplayer().handle_events(events=[{"invalid": "event"}]) self.assertIsNone(result) diff --git a/tests/cloud/pub_sub/test_events.py b/tests/cloud/pub_sub/test_events.py index 9168d3cc0..b5c56811e 100644 --- a/tests/cloud/pub_sub/test_events.py +++ b/tests/cloud/pub_sub/test_events.py @@ -697,7 +697,6 @@ def test_pull_and_enqueue_available_events(self): event_handler._pull_and_enqueue_available_events(timeout=10) self.assertEqual(event_handler.waiting_events, {0: mock_message}) - self.assertEqual(event_handler._earliest_waiting_event_number, 0) def test_timeout_error_raised_if_result_message_not_received_in_time(self): """Test that a timeout error is raised if a result message is not received in time.""" From f1992cad35d7194796c9a212d982928de00269f7 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 11 Apr 2024 10:35:51 +0100 Subject: [PATCH 151/169] ENH: Add `datetime` attribute to all events skipci --- octue/cloud/pub_sub/service.py | 1 + 1 file changed, 1 insertion(+) diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index 5abd17ac4..054d8c255 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -467,6 +467,7 @@ def _emit_event(self, event, originator, recipient, order, attributes=None, time with emit_event_lock: attributes["order"] = int(order) + attributes["datetime"] = datetime.datetime.utcnow().isoformat() converted_attributes = {} for key, value in attributes.items(): From 8448560cf6a371cadd695dcb0313a166ba7b8cd5 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 11 Apr 2024 11:22:05 +0100 Subject: [PATCH 152/169] OPS: Upgrade to latest event store format --- terraform/bigquery.tf | 10 ++++++++++ terraform/functions.tf | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/terraform/bigquery.tf b/terraform/bigquery.tf index 4698ed662..63abffe09 100644 --- a/terraform/bigquery.tf +++ b/terraform/bigquery.tf @@ -15,6 +15,16 @@ resource "google_bigquery_table" "test_table" { schema = < Date: Thu, 11 Apr 2024 11:48:22 +0100 Subject: [PATCH 153/169] ENH: Use latest services communication schema BREAKING CHANGE: Upgrade all services in your network to this version of `octue` or above. --- octue/cloud/emulators/_pub_sub.py | 2 ++ octue/cloud/events/validation.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/octue/cloud/emulators/_pub_sub.py b/octue/cloud/emulators/_pub_sub.py index 09e80f27c..ee6733ca5 100644 --- a/octue/cloud/emulators/_pub_sub.py +++ b/octue/cloud/emulators/_pub_sub.py @@ -377,6 +377,8 @@ def ask( MockMessage.from_primitive( data=question, attributes={ + "datetime": "2024-04-11T10:46:48.236064", + "uuid": "a9de11b1-e88f-43fa-b3a4-40a590c3443f", "question_uuid": question_uuid, "forward_logs": subscribe_to_logs, "save_diagnostics": save_diagnostics, diff --git a/octue/cloud/events/validation.py b/octue/cloud/events/validation.py index c79594965..422b42163 100644 --- a/octue/cloud/events/validation.py +++ b/octue/cloud/events/validation.py @@ -8,7 +8,7 @@ VALID_EVENT_KINDS = {"delivery_acknowledgement", "heartbeat", "log_record", "monitor_message", "exception", "result"} -SERVICE_COMMUNICATION_SCHEMA = {"$ref": "https://jsonschema.registry.octue.com/octue/service-communication/0.9.0.json"} +SERVICE_COMMUNICATION_SCHEMA = {"$ref": "https://jsonschema.registry.octue.com/octue/service-communication/0.10.0.json"} SERVICE_COMMUNICATION_SCHEMA_INFO_URL = "https://strands.octue.com/octue/service-communication" SERVICE_COMMUNICATION_SCHEMA_VERSION = os.path.splitext(SERVICE_COMMUNICATION_SCHEMA["$ref"])[0].split("/")[-1] From 480f0ae43e81dc492b5d499dfa4861d99de1b97a Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 11 Apr 2024 12:00:24 +0100 Subject: [PATCH 154/169] ENH: Add new event store fields to `get_events` function --- octue/cloud/pub_sub/bigquery.py | 2 ++ tests/cloud/pub_sub/test_bigquery.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/octue/cloud/pub_sub/bigquery.py b/octue/cloud/pub_sub/bigquery.py index ccb2d9beb..d02833c07 100644 --- a/octue/cloud/pub_sub/bigquery.py +++ b/octue/cloud/pub_sub/bigquery.py @@ -39,6 +39,8 @@ def get_events( if include_attributes: fields.extend( ( + "`datetime`", + "`uuid`", "`originator`", "`sender`", "`sender_type`", diff --git a/tests/cloud/pub_sub/test_bigquery.py b/tests/cloud/pub_sub/test_bigquery.py index c34e6a403..bf019bb78 100644 --- a/tests/cloud/pub_sub/test_bigquery.py +++ b/tests/cloud/pub_sub/test_bigquery.py @@ -49,7 +49,7 @@ def test_with_attributes(self): self.assertEqual( mock_client.mock_calls[1].args[0], - "SELECT `event`, `originator`, `sender`, `sender_type`, `sender_sdk_version`, `recipient`, `order`, `other_attributes` FROM `blah`\n" + "SELECT `event`, `datetime`, `uuid`, `originator`, `sender`, `sender_type`, `sender_sdk_version`, `recipient`, `order`, `other_attributes` FROM `blah`\n" "WHERE sender=@sender\nAND question_uuid=@question_uuid\nORDER BY `order`\nLIMIT @limit", ) From a5930e4fc4e897ec8329ba488b33e22ae3b5efbf Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 11 Apr 2024 12:24:44 +0100 Subject: [PATCH 155/169] REF: Update `ChildEmulator` to use `event` instead of `message` BREAKING CHANGE: Replace any usage of the word `message` to `event` in any arguments, method names, and files containing events. --- octue/cloud/emulators/child.py | 115 +++++++++--------- octue/runner.py | 2 +- octue/utils/testing.py | 2 +- tests/cloud/emulators/test_child_emulator.py | 114 ++++++++--------- .../file_with_exception.json | 2 +- ...ssages.json => file_with_only_events.json} | 2 +- .../file_with_output_manifest.json | 2 +- .../valid_child_emulator_files/full_file.json | 2 +- tests/data/diagnostics/questions.json | 2 +- tests/templates/test_template_apps.py | 4 +- tests/test_cli.py | 4 +- tests/test_runner.py | 34 ++++-- 12 files changed, 145 insertions(+), 140 deletions(-) rename tests/cloud/emulators/valid_child_emulator_files/{file_with_only_messages.json => file_with_only_events.json} (95%) diff --git a/octue/cloud/emulators/child.py b/octue/cloud/emulators/child.py index 02d33d409..5f60759b3 100644 --- a/octue/cloud/emulators/child.py +++ b/octue/cloud/emulators/child.py @@ -13,19 +13,19 @@ class ChildEmulator: - """An emulator for the `octue.resources.child.Child` class that sends the given messages to the parent for handling - without contacting the real child or using Pub/Sub. Any messages a real child could produce are supported. `Child` + """An emulator for the `octue.resources.child.Child` class that sends the given events to the parent for handling + without contacting the real child or using Pub/Sub. Any events a real child could produce are supported. `Child` instances can be replaced/mocked like-for-like by `ChildEmulator` without the parent knowing. :param str|None id: the ID of the child; a UUID is generated if none is provided :param dict|None backend: a dictionary including the key "name" with a value of the name of the type of backend (e.g. "GCPPubSubBackend") and key-value pairs for any other parameters the chosen backend expects; a mock backend is used if none is provided :param str internal_service_name: the name to give to the internal service used to ask questions to the child - :param list(dict)|None messages: the list of messages to send to the parent + :param list(dict)|None events: the list of events to send to the parent :return None: """ - def __init__(self, id=None, backend=None, internal_service_name="local/local:local", messages=None): - self.messages = messages or [] + def __init__(self, id=None, backend=None, internal_service_name="local/local:local", events=None): + self.events = events or [] backend = copy.deepcopy(backend or {"name": "GCPPubSubBackend", "project_name": "emulated-project"}) backend_type_name = backend.pop("name") @@ -40,7 +40,7 @@ def __init__(self, id=None, backend=None, internal_service_name="local/local:loc children={self._child.id: self._child}, ) - self._message_handlers = { + self._event_handlers = { "delivery_acknowledgement": self._handle_delivery_acknowledgement, "heartbeat": self._handle_heartbeat, "log_record": self._handle_log_record, @@ -49,7 +49,7 @@ def __init__(self, id=None, backend=None, internal_service_name="local/local:loc "result": self._handle_result, } - self._valid_message_kinds = set(self._message_handlers.keys()) + self._valid_event_kinds = set(self._event_handlers.keys()) @classmethod def from_file(cls, path): @@ -66,7 +66,7 @@ def from_file(cls, path): id=serialised_child_emulator.get("id"), backend=serialised_child_emulator.get("backend"), internal_service_name=serialised_child_emulator.get("internal_service_name"), - messages=serialised_child_emulator.get("messages"), + events=serialised_child_emulator.get("events"), ) def __repr__(self): @@ -97,9 +97,9 @@ def ask( asynchronous=False, timeout=86400, ): - """Ask the child emulator a question and receive its emulated response messages. Unlike a real child, the input + """Ask the child emulator a question and receive its emulated response events. Unlike a real child, the input values and manifest are not validated against the schema in the child's twine as it is only available to the - real child. Hence, the input values and manifest do not affect the messages returned by the emulator. + real child. Hence, the input values and manifest do not affect the events returned by the emulator. :param any|None input_values: any input values for the question :param octue.resources.manifest.Manifest|None input_manifest: an input manifest of any datasets needed for the question @@ -145,7 +145,7 @@ def _emulate_analysis( handle_monitor_message, save_diagnostics, ): - """Emulate analysis of a question by handling the messages given at instantiation in the order given. + """Emulate analysis of a question by handling the events given at instantiation in the order given. :param str|None analysis_id: UUID of analysis :param str|dict|None input_values: any input values for the question @@ -156,12 +156,12 @@ def _emulate_analysis( :param str save_diagnostics: must be one of {"SAVE_DIAGNOSTICS_OFF", "SAVE_DIAGNOSTICS_ON_CRASH", "SAVE_DIAGNOSTICS_ON"}; if turned on, allow the input values and manifest (and its datasets) to be saved by the child either all the time or just if the analysis fails :return octue.resources.analysis.Analysis: """ - for message in self.messages: - self._validate_message(message) - handler = self._message_handlers[message["kind"]] + for event in self.events: + self._validate_event(event) + handler = self._event_handlers[event["kind"]] result = handler( - message, + event, analysis_id=analysis_id, input_values=input_values, input_manifest=input_manifest, @@ -174,7 +174,7 @@ def _emulate_analysis( if result: return result - # If no result message is included in the given messages, return an empty analysis. + # If no result event is included in the given events, return an empty analysis. return Analysis( id=analysis_id, twine={}, @@ -185,59 +185,56 @@ def _emulate_analysis( output_manifest=None, ) - def _validate_message(self, message): - """Validate the given message to ensure it can be handled. + def _validate_event(self, event): + """Validate the given event to ensure it can be handled. - :param dict message: - :raise TypeError: if the message isn't a dictionary - :raise ValueError: if the message doesn't contain a 'kind' key or if the 'kind' key maps to an invalid value + :param dict event: + :raise TypeError: if the event isn't a dictionary + :raise ValueError: if the event doesn't contain a 'kind' key or if the 'kind' key maps to an invalid value :return None: """ - if not isinstance(message, dict): - raise TypeError("Each message must be a dictionary.") + if not isinstance(event, dict): + raise TypeError("Each event must be a dictionary.") - if "kind" not in message: - raise ValueError( - f"Each message must contain a 'kind' key mapping to one of: {self._valid_message_kinds!r}." - ) + if "kind" not in event: + raise ValueError(f"Each event must contain a 'kind' key mapping to one of: {self._valid_event_kinds!r}.") - if message["kind"] not in self._valid_message_kinds: + if event["kind"] not in self._valid_event_kinds: raise ValueError( - f"{message['kind']!r} is an invalid message kind for the ChildEmulator. The valid kinds are: " - f"{self._valid_message_kinds!r}." + f"{event['kind']!r} is an invalid event kind for the ChildEmulator. The valid kinds are: " + f"{self._valid_event_kinds!r}." ) - def _handle_delivery_acknowledgement(self, message, **kwargs): - """A no-operation handler for delivery acknowledgement messages (these messages are ignored by the child - emulator). + def _handle_delivery_acknowledgement(self, event, **kwargs): + """A no-operation handler for delivery acknowledgement events (these events are ignored by the child emulator). - :param dict message: a dictionary containing the key "datetime" + :param dict event: a dictionary containing the key "datetime" :param kwargs: this should be empty :return None: """ - logger.warning("Delivery acknowledgement messages are ignored by the ChildEmulator.") + logger.warning("Delivery acknowledgement events are ignored by the ChildEmulator.") - def _handle_heartbeat(self, message, **kwargs): - """A no-operation handler for heartbeat messages (these messages are ignored by the child emulator). + def _handle_heartbeat(self, event, **kwargs): + """A no-operation handler for heartbeat events (these events are ignored by the child emulator). - :param dict message: a dictionary containing the key "datetime" + :param dict event: a dictionary containing the key "datetime" :param kwargs: this should be empty :return None: """ - logger.warning("Heartbeat messages are ignored by the ChildEmulator.") + logger.warning("Heartbeat events are ignored by the ChildEmulator.") - def _handle_log_record(self, message, **kwargs): - """Convert the given message into a log record and pass it to the log handler. + def _handle_log_record(self, event, **kwargs): + """Convert the given event into a log record and pass it to the log handler. - :param dict message: a dictionary containing a "log_record" key whose value is a dictionary representing a log record + :param dict event: a dictionary containing a "log_record" key whose value is a dictionary representing a log record :param kwargs: this should be empty - :raise TypeError: if the message can't be converted to a log record + :raise TypeError: if the event can't be converted to a log record :return None: """ try: - log_record = message["log_record"] + log_record = event["log_record"] except KeyError: - raise ValueError("Log record messages must include a 'log_record' key.") + raise ValueError("Log record events must include a 'log_record' key.") try: log_record["levelno"] = log_record.get("levelno", 20) @@ -247,52 +244,52 @@ def _handle_log_record(self, message, **kwargs): except Exception: raise TypeError( - "The 'log_record' key in a log record message must map to a dictionary that can be converted by " + "The 'log_record' key in a log record event must map to a dictionary that can be converted by " "`logging.makeLogRecord` to a `logging.LogRecord` instance." ) - def _handle_monitor_message(self, message, **kwargs): + def _handle_monitor_message(self, event, **kwargs): """Handle a monitor message with the given handler. - :param dict message: a dictionary containing a "data" key mapped to a JSON-encoded string representing a monitor message. This monitor message will be handled by the monitor message handler + :param dict event: a dictionary containing a "data" key mapped to a JSON-encoded string representing a monitor message. This monitor message will be handled by the monitor message handler :param kwargs: must include the "handle_monitor_message" key :return None: """ - kwargs.get("handle_monitor_message")(message["data"]) + kwargs.get("handle_monitor_message")(event["data"]) - def _handle_exception(self, message, **kwargs): + def _handle_exception(self, event, **kwargs): """Raise the given exception. - :param dict message: a dictionary representing the exception to be raised; it must include the "exception_type" and "exception_message" keys + :param dict event: a dictionary representing the exception to be raised; it must include the "exception_type" and "exception_message" keys :param kwargs: this should be empty :raise ValueError: if the given exception cannot be raised :return None: """ - if "exception_type" not in message or "exception_message" not in message: + if "exception_type" not in event or "exception_message" not in event: raise ValueError( "The exception must be given as a dictionary containing the keys 'exception_type' and " "'exception_message'." ) try: - exception_type = EXCEPTIONS_MAPPING[message["exception_type"]] + exception_type = EXCEPTIONS_MAPPING[event["exception_type"]] # Allow unknown exception types to still be raised. except KeyError: - exception_type = type(message["exception_type"], (Exception,), {}) + exception_type = type(event["exception_type"], (Exception,), {}) - raise exception_type(message["exception_message"]) + raise exception_type(event["exception_message"]) - def _handle_result(self, message, **kwargs): + def _handle_result(self, event, **kwargs): """Return the result as an `Analysis` instance. - :param dict message: a dictionary containing an "output_values" key and an "output_manifest" key + :param dict event: a dictionary containing an "output_values" key and an "output_manifest" key :param kwargs: must contain the keys "analysis_id", "handle_monitor_message", "input_values", and "input_manifest" :raise ValueError: if the result doesn't contain the "output_values" and "output_manifest" keys :return octue.resources.analysis.Analysis: an `Analysis` instance containing the emulated outputs """ input_manifest = kwargs.get("input_manifest") - output_manifest = message.get("output_manifest") + output_manifest = event.get("output_manifest") if input_manifest and not isinstance(input_manifest, Manifest): input_manifest = Manifest.deserialise(input_manifest) @@ -306,7 +303,7 @@ def _handle_result(self, message, **kwargs): handle_monitor_message=kwargs["handle_monitor_message"], input_values=kwargs["input_values"], input_manifest=input_manifest, - output_values=message.get("output_values"), + output_values=event.get("output_values"), output_manifest=output_manifest, ) diff --git a/octue/runner.py b/octue/runner.py index c65d6e815..5228429f5 100644 --- a/octue/runner.py +++ b/octue/runner.py @@ -374,7 +374,7 @@ def wrapper(*args, **kwargs): try: return original_ask_method(**kwargs) finally: - self.diagnostics.add_question({"id": child.id, "key": key, **kwargs, "messages": child.received_events}) + self.diagnostics.add_question({"id": child.id, "key": key, **kwargs, "events": child.received_events}) return wrapper diff --git a/octue/utils/testing.py b/octue/utils/testing.py index b1bc8deb6..270df0ed7 100644 --- a/octue/utils/testing.py +++ b/octue/utils/testing.py @@ -65,4 +65,4 @@ def _load_child_emulators(path): with open(os.path.join(path, "questions.json")) as f: questions = json.load(f) - return tuple(ChildEmulator(id=question["id"], messages=question["messages"]) for question in questions) + return tuple(ChildEmulator(id=question["id"], events=question["events"]) for question in questions) diff --git a/tests/cloud/emulators/test_child_emulator.py b/tests/cloud/emulators/test_child_emulator.py index 55cf5bea2..1310d6223 100644 --- a/tests/cloud/emulators/test_child_emulator.py +++ b/tests/cloud/emulators/test_child_emulator.py @@ -30,35 +30,35 @@ def test_representation(self): f"", ) - def test_ask_with_non_dictionary_message(self): - """Test that messages that aren't dictionaries fail validation.""" - messages = [ + def test_ask_with_non_dictionary_event(self): + """Test that events that aren't dictionaries fail validation.""" + events = [ ["hello"], ] - child_emulator = ChildEmulator(backend=self.BACKEND, messages=messages) + child_emulator = ChildEmulator(backend=self.BACKEND, events=events) with self.assertRaises(TypeError): child_emulator.ask(input_values={"hello": "world"}) - def test_ask_with_invalid_message_structure(self): - """Test that messages with an invalid structure fail validation.""" - messages = [ - {"message": "hello"}, + def test_ask_with_invalid_event_structure(self): + """Test that events with an invalid structure fail validation.""" + events = [ + {"event": "hello"}, ] - child_emulator = ChildEmulator(backend=self.BACKEND, messages=messages) + child_emulator = ChildEmulator(backend=self.BACKEND, events=events) with self.assertRaises(ValueError): child_emulator.ask(input_values={"hello": "world"}) - def test_ask_with_invalid_message_type(self): - """Test that messages with an invalid type fail validation.""" - messages = [ + def test_ask_with_invalid_event_type(self): + """Test that events with an invalid type fail validation.""" + events = [ {"kind": "hello", "content": [1, 2, 3, 4]}, ] - child_emulator = ChildEmulator(backend=self.BACKEND, messages=messages) + child_emulator = ChildEmulator(backend=self.BACKEND, events=events) with self.assertRaises(ValueError): child_emulator.ask(input_values={"hello": "world"}) @@ -66,23 +66,23 @@ def test_ask_with_invalid_message_type(self): # Re-enable this when schema validation has been sorted out in the child emulator. # def test_ask_with_invalid_result(self): # """Test that an invalid result fails validation.""" - # messages = [ + # events = [ # { # "kind": "result", # "wrong": "keys", # }, # ] # - # child_emulator = ChildEmulator(backend=self.BACKEND, messages=messages) + # child_emulator = ChildEmulator(backend=self.BACKEND, events=events) # # with self.assertRaises(ValueError): # child_emulator.ask(input_values={"hello": "world"}) - def test_ask_with_result_message(self): - """Test that result messages are returned by the emulator's ask method.""" + def test_ask_with_result_event(self): + """Test that result events are returned by the emulator's ask method.""" output_manifest = Manifest() - messages = [ + events = [ { "kind": "result", "output_values": [1, 2, 3, 4], @@ -90,7 +90,7 @@ def test_ask_with_result_message(self): }, ] - child_emulator = ChildEmulator(backend=self.BACKEND, messages=messages) + child_emulator = ChildEmulator(backend=self.BACKEND, events=events) result = child_emulator.ask(input_values={"hello": "world"}) self.assertEqual(result["output_values"], [1, 2, 3, 4]) @@ -100,7 +100,7 @@ def test_ask_with_input_manifest(self): """Test that a child emulator can accept an input manifest.""" input_manifest = Manifest() - messages = [ + events = [ { "kind": "result", "output_values": [1, 2, 3, 4], @@ -108,47 +108,47 @@ def test_ask_with_input_manifest(self): }, ] - child_emulator = ChildEmulator(backend=self.BACKEND, messages=messages) + child_emulator = ChildEmulator(backend=self.BACKEND, events=events) result = child_emulator.ask(input_values={"hello": "world"}, input_manifest=input_manifest) self.assertEqual(result["output_values"], [1, 2, 3, 4]) - def test_empty_output_returned_by_ask_if_no_result_present_in_messages(self): - """Test that an empty output is returned if no result message is present in the given messages.""" - child_emulator = ChildEmulator(backend=self.BACKEND, messages=[]) + def test_empty_output_returned_by_ask_if_no_result_present_in_events(self): + """Test that an empty output is returned if no result event is present in the given events.""" + child_emulator = ChildEmulator(backend=self.BACKEND, events=[]) result = child_emulator.ask(input_values={"hello": "world"}) self.assertEqual(result, {"output_values": None, "output_manifest": None}) def test_ask_with_log_record_with_missing_log_record_key(self): - """Test that an error is raised if a log record message missing the "log_record" key is given.""" - messages = [ + """Test that an error is raised if a log record event missing the "log_record" key is given.""" + events = [ { "kind": "log_record", } ] - child_emulator = ChildEmulator(backend=self.BACKEND, messages=messages) + child_emulator = ChildEmulator(backend=self.BACKEND, events=events) with self.assertRaises(ValueError): child_emulator.ask(input_values={"hello": "world"}) def test_ask_with_invalid_log_record(self): """Test that an invalid log record representation fails validation.""" - messages = [ + events = [ { "kind": "log_record", "log_record": [1, 2, 3], }, ] - child_emulator = ChildEmulator(backend=self.BACKEND, messages=messages) + child_emulator = ChildEmulator(backend=self.BACKEND, events=events) with self.assertRaises(TypeError): child_emulator.ask(input_values={"hello": "world"}) def test_ask_with_logs(self): """Test that log records can be handled by the emulator.""" - messages = [ + events = [ { "kind": "log_record", "log_record": {"msg": "Starting analysis.", "levelno": 20, "levelname": "INFO"}, @@ -159,7 +159,7 @@ def test_ask_with_logs(self): }, ] - child_emulator = ChildEmulator(backend=self.BACKEND, messages=messages) + child_emulator = ChildEmulator(backend=self.BACKEND, events=events) with self.assertLogs(level=logging.INFO) as logging_context: child_emulator.ask(input_values={"hello": "world"}) @@ -169,7 +169,7 @@ def test_ask_with_logs(self): def test_ask_with_logs_without_level_number_and_name(self): """Test that the 'INFO' log level is used if none is provided in the log record dictionaries.""" - messages = [ + events = [ { "kind": "log_record", "log_record": {"msg": "Starting analysis."}, @@ -180,7 +180,7 @@ def test_ask_with_logs_without_level_number_and_name(self): }, ] - child_emulator = ChildEmulator(backend=self.BACKEND, messages=messages) + child_emulator = ChildEmulator(backend=self.BACKEND, events=events) with self.assertLogs(level=logging.INFO) as logging_context: child_emulator.ask(input_values={"hello": "world"}) @@ -193,21 +193,21 @@ def test_ask_with_logs_without_level_number_and_name(self): def test_ask_with_invalid_exception(self): """Test that an invalid exception fails validation.""" - messages = [ + events = [ { "kind": "exception", "not": "an exception", }, ] - child_emulator = ChildEmulator(backend=self.BACKEND, messages=messages) + child_emulator = ChildEmulator(backend=self.BACKEND, events=events) with self.assertRaises(ValueError): child_emulator.ask(input_values={"hello": "world"}) def test_ask_with_exception(self): """Test that exceptions are raised by the emulator.""" - messages = [ + events = [ { "kind": "exception", "exception_type": "TypeError", @@ -215,7 +215,7 @@ def test_ask_with_exception(self): }, ] - child_emulator = ChildEmulator(backend=self.BACKEND, messages=messages) + child_emulator = ChildEmulator(backend=self.BACKEND, events=events) # Test that the exception was raised. with self.assertRaises(TypeError) as context: @@ -226,14 +226,14 @@ def test_ask_with_exception(self): def test_ask_with_monitor_message(self): """Test that monitor messages are handled by the emulator.""" - messages = [ + events = [ { "kind": "monitor_message", "data": "A sample monitor message.", }, ] - child_emulator = ChildEmulator(backend=self.BACKEND, messages=messages) + child_emulator = ChildEmulator(backend=self.BACKEND, events=events) monitor_messages = [] @@ -245,25 +245,25 @@ def test_ask_with_monitor_message(self): # Check that the monitor message handler has worked. self.assertEqual(monitor_messages, ["A sample monitor message."]) - def test_heartbeat_messages_are_ignored(self): - """Test that heartbeat messages are ignored by the emulator.""" - messages = [ + def test_heartbeat_events_are_ignored(self): + """Test that heartbeat events are ignored by the emulator.""" + events = [ { "kind": "heartbeat", "datetime": "2023-11-23T14:25:38.142884", }, ] - child_emulator = ChildEmulator(backend=self.BACKEND, messages=messages) + child_emulator = ChildEmulator(backend=self.BACKEND, events=events) with self.assertLogs(level=logging.WARNING) as logging_context: child_emulator.ask(input_values={"hello": "world"}) - self.assertIn("Heartbeat messages are ignored by the ChildEmulator.", logging_context.output[0]) + self.assertIn("Heartbeat events are ignored by the ChildEmulator.", logging_context.output[0]) - def test_messages_recorded_from_real_child_can_be_used_in_child_emulator(self): - """Test that messages recorded from a real child can be used as emulated messages in a child emulator (i.e. test - that the message format is unified between `Service` and `ChildEmulator`). + def test_events_recorded_from_real_child_can_be_used_in_child_emulator(self): + """Test that events recorded from a real child can be used as emulated events in a child emulator (i.e. test + that the event format is unified between `Service` and `ChildEmulator`). """ backend = GCPPubSubBackend(project_name="my-project") @@ -280,14 +280,14 @@ def error_run_function(*args, **kwargs): subscription, _ = parent.ask(service_id=child.id, input_values={}) parent.wait_for_answer(subscription=subscription) - child_emulator = ChildEmulator(messages=parent.received_events) + child_emulator = ChildEmulator(events=parent.received_events) with self.assertRaises(OSError): child_emulator.ask(input_values={}) def test_ask_more_than_one_question(self): """Test than a child emulator can be asked more than one question without an error occurring.""" - messages = [ + events = [ { "kind": "result", "output_values": [1, 2, 3, 4], @@ -295,7 +295,7 @@ def test_ask_more_than_one_question(self): }, ] - child_emulator = ChildEmulator(backend=self.BACKEND, messages=messages) + child_emulator = ChildEmulator(backend=self.BACKEND, events=events) result_0 = child_emulator.ask(input_values={"hello": "world"}) result_1 = child_emulator.ask(input_values={"hello": "planet"}) self.assertEqual(result_0, result_1) @@ -320,11 +320,11 @@ def test_with_empty_file(self): result = child_emulator.ask(input_values={"hello": "world"}) self.assertEqual(result, {"output_values": None, "output_manifest": None}) - def test_with_only_messages(self): - """Test that a child emulator can be instantiated from a JSON file containing only the messages it should - produce, asked a question, and produce the expected log messages, monitor messages, and result. + def test_with_only_events(self): + """Test that a child emulator can be instantiated from a JSON file containing only the events it should + produce, asked a question, and produce the expected log events, monitor messages, and result. """ - emulator_file_path = os.path.join(self.TEST_FILES_DIRECTORY, "file_with_only_messages.json") + emulator_file_path = os.path.join(self.TEST_FILES_DIRECTORY, "file_with_only_events.json") child_emulator = ChildEmulator.from_file(emulator_file_path) self.assertEqual(child_emulator._child.backend.project_name, "emulated-project") @@ -348,7 +348,7 @@ def test_with_only_messages(self): def test_with_full_file(self): """Test that a child emulator can be instantiated from a JSON file containing all its instantiation arguments, - asked a question, and produce the correct messages. + asked a question, and produce the correct events. """ child_emulator = ChildEmulator.from_file(os.path.join(self.TEST_FILES_DIRECTORY, "full_file.json")) self.assertEqual(child_emulator.id, f"octue/my-child:{MOCK_SERVICE_REVISION_TAG}") @@ -374,8 +374,8 @@ def test_with_full_file(self): self.assertEqual(result, {"output_values": [1, 2, 3, 4, 5], "output_manifest": None}) def test_with_exception(self): - """Test that a child emulator can be instantiated from a JSON file including an exception in its messages and, - when asked a question, produce the messages prior to the exception before raising the exception. + """Test that a child emulator can be instantiated from a JSON file including an exception in its events and, + when asked a question, produce the events prior to the exception before raising the exception. """ child_emulator = ChildEmulator.from_file(os.path.join(self.TEST_FILES_DIRECTORY, "file_with_exception.json")) diff --git a/tests/cloud/emulators/valid_child_emulator_files/file_with_exception.json b/tests/cloud/emulators/valid_child_emulator_files/file_with_exception.json index dddcff096..cfe5a264b 100644 --- a/tests/cloud/emulators/valid_child_emulator_files/file_with_exception.json +++ b/tests/cloud/emulators/valid_child_emulator_files/file_with_exception.json @@ -1,5 +1,5 @@ { - "messages": [ + "events": [ { "kind": "log_record", "log_record": { "msg": "Starting analysis." } diff --git a/tests/cloud/emulators/valid_child_emulator_files/file_with_only_messages.json b/tests/cloud/emulators/valid_child_emulator_files/file_with_only_events.json similarity index 95% rename from tests/cloud/emulators/valid_child_emulator_files/file_with_only_messages.json rename to tests/cloud/emulators/valid_child_emulator_files/file_with_only_events.json index 1cacb36ff..0bbbc4a5c 100644 --- a/tests/cloud/emulators/valid_child_emulator_files/file_with_only_messages.json +++ b/tests/cloud/emulators/valid_child_emulator_files/file_with_only_events.json @@ -1,5 +1,5 @@ { - "messages": [ + "events": [ { "kind": "log_record", "log_record": { "msg": "Starting analysis." } diff --git a/tests/cloud/emulators/valid_child_emulator_files/file_with_output_manifest.json b/tests/cloud/emulators/valid_child_emulator_files/file_with_output_manifest.json index bd965a29f..596c193ae 100644 --- a/tests/cloud/emulators/valid_child_emulator_files/file_with_output_manifest.json +++ b/tests/cloud/emulators/valid_child_emulator_files/file_with_output_manifest.json @@ -1,5 +1,5 @@ { - "messages": [ + "events": [ { "kind": "result", "output_manifest": { diff --git a/tests/cloud/emulators/valid_child_emulator_files/full_file.json b/tests/cloud/emulators/valid_child_emulator_files/full_file.json index f16e56874..c3d0892b9 100644 --- a/tests/cloud/emulators/valid_child_emulator_files/full_file.json +++ b/tests/cloud/emulators/valid_child_emulator_files/full_file.json @@ -5,7 +5,7 @@ "project_name": "blah" }, "internal_service_name": "octue/my-service:2.3.0", - "messages": [ + "events": [ { "kind": "log_record", "log_record": { "msg": "Starting analysis." } diff --git a/tests/data/diagnostics/questions.json b/tests/data/diagnostics/questions.json index 212ae9b36..44edac3d7 100644 --- a/tests/data/diagnostics/questions.json +++ b/tests/data/diagnostics/questions.json @@ -1,7 +1,7 @@ [ { "id": "octue/my-child:2.3.0", - "messages": [ + "events": [ { "kind": "log_record", "log_record": { "msg": "Starting analysis." } diff --git a/tests/templates/test_template_apps.py b/tests/templates/test_template_apps.py index df5809710..d5b784268 100644 --- a/tests/templates/test_template_apps.py +++ b/tests/templates/test_template_apps.py @@ -147,7 +147,7 @@ def test_child_services_template_using_emulated_children(self): ChildEmulator( id=f"template-child-services/wind-speed-service:{MOCK_SERVICE_REVISION_TAG}", internal_service_name=runner.service_id, - messages=[ + events=[ {"kind": "log_record", "log_record": {"msg": "This is an emulated child log message."}}, {"kind": "result", "output_values": [10], "output_manifest": None}, ], @@ -155,7 +155,7 @@ def test_child_services_template_using_emulated_children(self): ChildEmulator( id=f"template-child-services/elevation-service:{MOCK_SERVICE_REVISION_TAG}", internal_service_name=runner.service_id, - messages=[ + events=[ {"kind": "result", "output_values": [300], "output_manifest": None}, ], ), diff --git a/tests/test_cli.py b/tests/test_cli.py index a2e661d93..c2e07a37e 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -315,7 +315,7 @@ def test_get_diagnostics(self): self.assertEqual(questions[0]["id"], f"octue/my-child:{MOCK_SERVICE_REVISION_TAG}") self.assertEqual( - questions[0]["messages"], + questions[0]["events"], [ {"kind": "log_record", "log_record": {"msg": "Starting analysis."}}, {"kind": "log_record", "log_record": {"msg": "Finishing analysis."}}, @@ -382,7 +382,7 @@ def test_get_diagnostics_with_datasets(self): self.assertEqual(questions[0]["id"], f"octue/my-child:{MOCK_SERVICE_REVISION_TAG}") self.assertEqual( - questions[0]["messages"], + questions[0]["events"], [ {"kind": "log_record", "log_record": {"msg": "Starting analysis."}}, {"kind": "log_record", "log_record": {"msg": "Finishing analysis."}}, diff --git a/tests/test_runner.py b/tests/test_runner.py index ddb7e040d..2895719e9 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -15,10 +15,13 @@ from octue import Runner, exceptions from octue.cloud import storage from octue.cloud.emulators import ChildEmulator +from octue.cloud.emulators._pub_sub import MockTopic +from octue.cloud.emulators.child import ServicePatcher +from octue.cloud.events import OCTUE_SERVICES_PREFIX from octue.cloud.storage import GoogleCloudStorageClient from octue.resources import Dataset, Manifest from octue.resources.datafile import Datafile -from tests import MOCK_SERVICE_REVISION_TAG, TEST_BUCKET_NAME, TESTS_DIR +from tests import MOCK_SERVICE_REVISION_TAG, TEST_BUCKET_NAME, TEST_PROJECT_NAME, TESTS_DIR from tests.base import BaseTestCase from tests.test_app_modules.app_class.app import App from tests.test_app_modules.app_module import app @@ -301,6 +304,11 @@ def app(analysis): def test_child_messages_saved_even_if_child_ask_method_raises_error(self): """Test that messages from the child are still saved even if an error is raised within the `Child.ask` method.""" + topic = MockTopic(name=OCTUE_SERVICES_PREFIX, project_name=TEST_PROJECT_NAME) + + with ServicePatcher(): + topic.create(allow_existing=True) + diagnostics_cloud_path = storage.path.generate_gs_path(TEST_BUCKET_NAME, "diagnostics") def app(analysis): @@ -341,13 +349,13 @@ def app(analysis): emulated_children = [ ChildEmulator( id=f"octue/the-child:{MOCK_SERVICE_REVISION_TAG}", - messages=[ + events=[ {"kind": "result", "output_values": [1, 4, 9, 16], "output_manifest": None}, ], ), ChildEmulator( id=f"octue/yet-another-child:{MOCK_SERVICE_REVISION_TAG}", - messages=[ + events=[ {"kind": "log_record", "log_record": {"msg": "Starting analysis."}}, {"kind": "log_record", "log_record": {"msg": "Finishing analysis."}}, { @@ -383,17 +391,17 @@ def app(analysis): self.assertEqual(questions[0]["key"], "my-child") self.assertEqual(questions[0]["id"], f"octue/the-child:{MOCK_SERVICE_REVISION_TAG}") self.assertEqual(questions[0]["input_values"], [1, 2, 3, 4]) - self.assertEqual(len(questions[0]["messages"]), 2) + self.assertEqual(len(questions[0]["events"]), 2) # Second question. self.assertEqual(questions[1]["key"], "another-child") self.assertEqual(questions[1]["id"], f"octue/yet-another-child:{MOCK_SERVICE_REVISION_TAG}") self.assertEqual(questions[1]["input_values"], "miaow") - self.assertEqual(questions[1]["messages"][1]["kind"], "exception") - self.assertEqual(questions[1]["messages"][1]["exception_type"], "ValueError") + self.assertEqual(questions[1]["events"][1]["kind"], "exception") + self.assertEqual(questions[1]["events"][1]["exception_type"], "ValueError") self.assertEqual( - questions[1]["messages"][1]["exception_message"], + questions[1]["events"][1]["exception_message"], f"Error in : Deliberately raised for " f"testing.", ) @@ -789,13 +797,13 @@ def app(analysis): emulated_children = [ ChildEmulator( id=f"octue/a-child:{MOCK_SERVICE_REVISION_TAG}", - messages=[ + events=[ {"kind": "result", "output_values": [1, 4, 9, 16], "output_manifest": None}, ], ), ChildEmulator( id=f"octue/another-child:{MOCK_SERVICE_REVISION_TAG}", - messages=[ + events=[ {"kind": "log_record", "log_record": {"msg": "Starting analysis."}}, {"kind": "log_record", "log_record": {"msg": "Finishing analysis."}}, {"kind": "result", "output_values": "woof", "output_manifest": None}, @@ -902,14 +910,14 @@ def app(analysis): self.assertEqual(questions[0]["key"], "my-child") self.assertEqual(questions[0]["id"], f"octue/a-child:{MOCK_SERVICE_REVISION_TAG}") self.assertEqual(questions[0]["input_values"], [1, 2, 3, 4]) - self.assertEqual(questions[0]["messages"][1]["output_values"], [1, 4, 9, 16]) - self.assertEqual(len(questions[0]["messages"]), 2) + self.assertEqual(questions[0]["events"][1]["output_values"], [1, 4, 9, 16]) + self.assertEqual(len(questions[0]["events"]), 2) # Second question. self.assertEqual(questions[1]["key"], "another-child") self.assertEqual(questions[1]["id"], f"octue/another-child:{MOCK_SERVICE_REVISION_TAG}") self.assertEqual(questions[1]["input_values"], "miaow") - self.assertEqual(questions[1]["messages"][1]["output_values"], "woof") + self.assertEqual(questions[1]["events"][1]["output_values"], "woof") # This should be 4 but log messages aren't currently being handled by the child emulator correctly. - self.assertEqual(len(questions[1]["messages"]), 2) + self.assertEqual(len(questions[1]["events"]), 2) From 2d708fddb4e6642de9367df727af76be1bf327d1 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 11 Apr 2024 14:06:32 +0100 Subject: [PATCH 156/169] TST: Create `octue.services` mock topic once per test run --- tests/base.py | 20 ++++++++++++++++++-- tests/cloud/emulators/test_child_emulator.py | 19 ++----------------- tests/cloud/pub_sub/test_events.py | 13 +++---------- tests/cloud/pub_sub/test_logging.py | 8 ++------ tests/cloud/pub_sub/test_service.py | 5 +---- tests/resources/test_child.py | 9 +++------ tests/test_cli.py | 8 +------- tests/test_runner.py | 10 +--------- 8 files changed, 31 insertions(+), 61 deletions(-) diff --git a/tests/base.py b/tests/base.py index c08037eb1..ab4084e83 100644 --- a/tests/base.py +++ b/tests/base.py @@ -5,10 +5,26 @@ import yaml from octue.cloud import storage +from octue.cloud.emulators._pub_sub import MockTopic +from octue.cloud.emulators.child import ServicePatcher from octue.cloud.emulators.cloud_storage import GoogleCloudStorageEmulatorTestResultModifier +from octue.cloud.events import OCTUE_SERVICES_PREFIX from octue.cloud.storage import GoogleCloudStorageClient from octue.resources import Datafile, Dataset, Manifest -from tests import TEST_BUCKET_NAME +from tests import TEST_BUCKET_NAME, TEST_PROJECT_NAME + + +class TestResultModifier(GoogleCloudStorageEmulatorTestResultModifier): + """A test result modifier based on `GoogleCloudStorageEmulatorTestResultModifier` that also creates a mock + `octue.services` topic. + """ + + def startTestRun(self): + super().startTestRun() + + with ServicePatcher(): + self.services_topic = MockTopic(name=OCTUE_SERVICES_PREFIX, project_name=TEST_PROJECT_NAME) + self.services_topic.create(allow_existing=True) class BaseTestCase(unittest.TestCase): @@ -16,7 +32,7 @@ class BaseTestCase(unittest.TestCase): - sets a path to the test data directory """ - test_result_modifier = GoogleCloudStorageEmulatorTestResultModifier(default_bucket_name=TEST_BUCKET_NAME) + test_result_modifier = TestResultModifier(default_bucket_name=TEST_BUCKET_NAME) setattr(unittest.TestResult, "startTestRun", test_result_modifier.startTestRun) setattr(unittest.TestResult, "stopTestRun", test_result_modifier.stopTestRun) diff --git a/tests/cloud/emulators/test_child_emulator.py b/tests/cloud/emulators/test_child_emulator.py index 1310d6223..755058f95 100644 --- a/tests/cloud/emulators/test_child_emulator.py +++ b/tests/cloud/emulators/test_child_emulator.py @@ -2,13 +2,12 @@ import os from octue.cloud import storage -from octue.cloud.emulators._pub_sub import MockService, MockTopic +from octue.cloud.emulators._pub_sub import MockService from octue.cloud.emulators.child import ChildEmulator, ServicePatcher -from octue.cloud.events import OCTUE_SERVICES_PREFIX from octue.cloud.storage import GoogleCloudStorageClient from octue.resources import Manifest from octue.resources.service_backends import GCPPubSubBackend -from tests import MOCK_SERVICE_REVISION_TAG, TEST_BUCKET_NAME, TEST_PROJECT_NAME, TESTS_DIR +from tests import MOCK_SERVICE_REVISION_TAG, TEST_BUCKET_NAME, TESTS_DIR from tests.base import BaseTestCase @@ -16,13 +15,6 @@ class TestChildEmulatorAsk(BaseTestCase): BACKEND = {"name": "GCPPubSubBackend", "project_name": "blah"} - @classmethod - def setUpClass(cls): - topic = MockTopic(name=OCTUE_SERVICES_PREFIX, project_name=TEST_PROJECT_NAME) - - with ServicePatcher(): - topic.create(allow_existing=True) - def test_representation(self): """Test that child emulators are represented correctly.""" self.assertEqual( @@ -305,13 +297,6 @@ class TestChildEmulatorJSONFiles(BaseTestCase): TEST_FILES_DIRECTORY = os.path.join(TESTS_DIR, "cloud", "emulators", "valid_child_emulator_files") - @classmethod - def setUpClass(cls): - topic = MockTopic(name=OCTUE_SERVICES_PREFIX, project_name=TEST_PROJECT_NAME) - - with ServicePatcher(): - topic.create(allow_existing=True) - def test_with_empty_file(self): """Test that a child emulator can be instantiated from an empty JSON file (a JSON file with only an empty object in), asked a question, and produce a trivial result. diff --git a/tests/cloud/pub_sub/test_events.py b/tests/cloud/pub_sub/test_events.py index b5c56811e..e7b32ccb7 100644 --- a/tests/cloud/pub_sub/test_events.py +++ b/tests/cloud/pub_sub/test_events.py @@ -3,9 +3,8 @@ import uuid from unittest.mock import patch -from octue.cloud.emulators._pub_sub import MESSAGES, MockMessage, MockService, MockSubscription, MockTopic +from octue.cloud.emulators._pub_sub import MESSAGES, MockMessage, MockService, MockSubscription from octue.cloud.emulators.child import ServicePatcher -from octue.cloud.events import OCTUE_SERVICES_PREFIX from octue.cloud.pub_sub.events import GoogleCloudPubSubEventHandler from octue.resources.service_backends import GCPPubSubBackend from tests import TEST_PROJECT_NAME @@ -20,12 +19,9 @@ def setUpClass(cls): cls.service_patcher.start() cls.question_uuid = str(uuid.uuid4()) - cls.topic = MockTopic(name=OCTUE_SERVICES_PREFIX, project_name=TEST_PROJECT_NAME) - cls.topic.create(allow_existing=True) - cls.subscription = MockSubscription( name=f"octue.services.my-org.my-service.1-0-0.answers.{cls.question_uuid}", - topic=cls.topic, + topic=cls.test_result_modifier.services_topic, ) cls.subscription.create() @@ -641,12 +637,9 @@ def setUpClass(cls): cls.service_patcher.start() cls.question_uuid = str(uuid.uuid4()) - cls.topic = MockTopic(name=OCTUE_SERVICES_PREFIX, project_name=TEST_PROJECT_NAME) - cls.topic.create(allow_existing=True) - cls.subscription = MockSubscription( name=f"my-org.my-service.1-0-0.answers.{cls.question_uuid}", - topic=cls.topic, + topic=cls.test_result_modifier.services_topic, ) cls.subscription.create() diff --git a/tests/cloud/pub_sub/test_logging.py b/tests/cloud/pub_sub/test_logging.py index 2f26af05d..77a8a76c2 100644 --- a/tests/cloud/pub_sub/test_logging.py +++ b/tests/cloud/pub_sub/test_logging.py @@ -3,13 +3,11 @@ from logging import makeLogRecord from unittest.mock import patch -from octue.cloud.emulators._pub_sub import MESSAGES, MockService, MockTopic +from octue.cloud.emulators._pub_sub import MESSAGES, MockService from octue.cloud.emulators.child import ServicePatcher -from octue.cloud.events import OCTUE_SERVICES_PREFIX from octue.cloud.events.counter import EventCounter from octue.cloud.pub_sub.logging import GoogleCloudPubSubHandler from octue.resources.service_backends import GCPPubSubBackend -from tests import TEST_PROJECT_NAME from tests.base import BaseTestCase @@ -23,13 +21,11 @@ class TestGoogleCloudPubSubHandler(BaseTestCase): @classmethod def setUpClass(cls): - """Start the service patcher and create a mock services topic. + """Start the service patcher. :return None: """ cls.service_patcher.start() - topic = MockTopic(name=OCTUE_SERVICES_PREFIX, project_name=TEST_PROJECT_NAME) - topic.create(allow_existing=True) @classmethod def tearDownClass(cls): diff --git a/tests/cloud/pub_sub/test_service.py b/tests/cloud/pub_sub/test_service.py index c0b10889c..8e6526acb 100644 --- a/tests/cloud/pub_sub/test_service.py +++ b/tests/cloud/pub_sub/test_service.py @@ -21,7 +21,6 @@ ) from octue.cloud.emulators.child import ServicePatcher from octue.cloud.emulators.cloud_storage import mock_generate_signed_url -from octue.cloud.events import OCTUE_SERVICES_PREFIX from octue.cloud.pub_sub.service import Service from octue.exceptions import InvalidMonitorMessage from octue.resources import Analysis, Datafile, Dataset, Manifest @@ -45,13 +44,11 @@ class TestService(BaseTestCase): @classmethod def setUpClass(cls): - """Start the service patcher and create a mock services topic. + """Start the service patcher. :return None: """ cls.service_patcher.start() - topic = MockTopic(name=OCTUE_SERVICES_PREFIX, project_name=TEST_PROJECT_NAME) - topic.create(allow_existing=True) @classmethod def tearDownClass(cls): diff --git a/tests/resources/test_child.py b/tests/resources/test_child.py index 502dcc940..c0e866736 100644 --- a/tests/resources/test_child.py +++ b/tests/resources/test_child.py @@ -8,12 +8,11 @@ from google.auth.exceptions import DefaultCredentialsError -from octue.cloud.emulators._pub_sub import MockAnalysis, MockService, MockTopic +from octue.cloud.emulators._pub_sub import MockAnalysis, MockService from octue.cloud.emulators.child import ServicePatcher -from octue.cloud.events import OCTUE_SERVICES_PREFIX from octue.resources.child import Child from octue.resources.service_backends import GCPPubSubBackend -from tests import MOCK_SERVICE_REVISION_TAG, TEST_PROJECT_NAME +from tests import MOCK_SERVICE_REVISION_TAG from tests.base import BaseTestCase @@ -37,13 +36,11 @@ class TestChild(BaseTestCase): @classmethod def setUpClass(cls): - """Start the service patcher and create a mock services topic. + """Start the service patcher.. :return None: """ cls.service_patcher.start() - topic = MockTopic(name=OCTUE_SERVICES_PREFIX, project_name=TEST_PROJECT_NAME) - topic.create(allow_existing=True) @classmethod def tearDownClass(cls): diff --git a/tests/test_cli.py b/tests/test_cli.py index c2e07a37e..153e2eff0 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -11,11 +11,10 @@ from octue.cloud import storage from octue.cloud.emulators._pub_sub import MockService, MockTopic from octue.cloud.emulators.child import ServicePatcher -from octue.cloud.events import OCTUE_SERVICES_PREFIX from octue.configuration import AppConfiguration, ServiceConfiguration from octue.resources import Dataset from octue.utils.patches import MultiPatcher -from tests import MOCK_SERVICE_REVISION_TAG, TEST_BUCKET_NAME, TEST_PROJECT_NAME, TESTS_DIR +from tests import MOCK_SERVICE_REVISION_TAG, TEST_BUCKET_NAME, TESTS_DIR from tests.base import BaseTestCase @@ -195,11 +194,6 @@ def setUpClass(cls): }, ) - topic = MockTopic(name=OCTUE_SERVICES_PREFIX, project_name=TEST_PROJECT_NAME) - - with ServicePatcher(): - topic.create(allow_existing=True) - def test_start_command(self): """Test that the start command works without error and uses the revision tag supplied in the `OCTUE_SERVICE_REVISION_TAG` environment variable. diff --git a/tests/test_runner.py b/tests/test_runner.py index 2895719e9..86fba1372 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -15,13 +15,10 @@ from octue import Runner, exceptions from octue.cloud import storage from octue.cloud.emulators import ChildEmulator -from octue.cloud.emulators._pub_sub import MockTopic -from octue.cloud.emulators.child import ServicePatcher -from octue.cloud.events import OCTUE_SERVICES_PREFIX from octue.cloud.storage import GoogleCloudStorageClient from octue.resources import Dataset, Manifest from octue.resources.datafile import Datafile -from tests import MOCK_SERVICE_REVISION_TAG, TEST_BUCKET_NAME, TEST_PROJECT_NAME, TESTS_DIR +from tests import MOCK_SERVICE_REVISION_TAG, TEST_BUCKET_NAME, TESTS_DIR from tests.base import BaseTestCase from tests.test_app_modules.app_class.app import App from tests.test_app_modules.app_module import app @@ -304,11 +301,6 @@ def app(analysis): def test_child_messages_saved_even_if_child_ask_method_raises_error(self): """Test that messages from the child are still saved even if an error is raised within the `Child.ask` method.""" - topic = MockTopic(name=OCTUE_SERVICES_PREFIX, project_name=TEST_PROJECT_NAME) - - with ServicePatcher(): - topic.create(allow_existing=True) - diagnostics_cloud_path = storage.path.generate_gs_path(TEST_BUCKET_NAME, "diagnostics") def app(analysis): From 2677114e204bfd4b8a0f0fc49c6b024dd13df9f3 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 11 Apr 2024 14:10:32 +0100 Subject: [PATCH 157/169] TST: Fix tests --- tests/cloud/events/test_replayer.py | 2 ++ tests/data/events.json | 16 ++++++++++++++++ tests/utils/test_testing.py | 2 +- 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/cloud/events/test_replayer.py b/tests/cloud/events/test_replayer.py index 368ca8a99..f9c16b4b5 100644 --- a/tests/cloud/events/test_replayer.py +++ b/tests/cloud/events/test_replayer.py @@ -33,6 +33,8 @@ def test_no_result_event(self): "kind": "delivery_acknowledgement", }, "attributes": { + "datetime": "2024-04-11T10:46:48.236064", + "uuid": "a9de11b1-e88f-43fa-b3a4-40a590c3443f", "order": "0", "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", "originator": "octue/test-service:1.0.0", diff --git a/tests/data/events.json b/tests/data/events.json index 4cd741a59..b170a47e7 100644 --- a/tests/data/events.json +++ b/tests/data/events.json @@ -5,6 +5,8 @@ "kind": "delivery_acknowledgement" }, "attributes": { + "datetime": "2024-04-11T10:46:48.236064", + "uuid": "a9de11b1-e88f-43fa-b3a4-40a590c3443f", "order": "0", "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", "originator": "octue/test-service:1.0.0", @@ -41,6 +43,8 @@ } }, "attributes": { + "datetime": "2024-04-11T10:46:48.236064", + "uuid": "a9de11b1-e88f-43fa-b3a4-40a590c3443f", "order": "1", "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", "originator": "octue/test-service:1.0.0", @@ -77,6 +81,8 @@ } }, "attributes": { + "datetime": "2024-04-11T10:46:48.236064", + "uuid": "a9de11b1-e88f-43fa-b3a4-40a590c3443f", "order": "2", "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", "originator": "octue/test-service:1.0.0", @@ -113,6 +119,8 @@ } }, "attributes": { + "datetime": "2024-04-11T10:46:48.236064", + "uuid": "a9de11b1-e88f-43fa-b3a4-40a590c3443f", "order": "3", "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", "originator": "octue/test-service:1.0.0", @@ -149,6 +157,8 @@ } }, "attributes": { + "datetime": "2024-04-11T10:46:48.236064", + "uuid": "a9de11b1-e88f-43fa-b3a4-40a590c3443f", "order": "4", "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", "originator": "octue/test-service:1.0.0", @@ -185,6 +195,8 @@ } }, "attributes": { + "datetime": "2024-04-11T10:46:48.236064", + "uuid": "a9de11b1-e88f-43fa-b3a4-40a590c3443f", "order": "5", "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", "originator": "octue/test-service:1.0.0", @@ -216,6 +228,8 @@ "output_values": [1, 2, 3, 4, 5] }, "attributes": { + "datetime": "2024-04-11T10:46:48.236064", + "uuid": "a9de11b1-e88f-43fa-b3a4-40a590c3443f", "order": "6", "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", "originator": "octue/test-service:1.0.0", @@ -231,6 +245,8 @@ "kind": "heartbeat" }, "attributes": { + "datetime": "2024-04-11T10:46:48.236064", + "uuid": "a9de11b1-e88f-43fa-b3a4-40a590c3443f", "order": "7", "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", "originator": "octue/test-service:1.0.0", diff --git a/tests/utils/test_testing.py b/tests/utils/test_testing.py index 26030c20f..eb058132c 100644 --- a/tests/utils/test_testing.py +++ b/tests/utils/test_testing.py @@ -28,7 +28,7 @@ def test_load_test_fixture_from_downloaded_diagnostics(self): self.assertEqual(len(child_emulators), 1) self.assertEqual(child_emulators[0].id, f"octue/my-child:{MOCK_SERVICE_REVISION_TAG}") self.assertEqual( - child_emulators[0].messages[2:], + child_emulators[0].events[2:], [ {"kind": "monitor_message", "data": {"sample": "data"}}, {"kind": "result", "output_values": [1, 2, 3, 4, 5]}, From b8d26fb07c820a03aa6de98d9d155a5b1a0a5d48 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 11 Apr 2024 14:30:35 +0100 Subject: [PATCH 158/169] TST: Update test class names --- tests/cloud/events/test_replayer.py | 2 +- tests/cloud/pub_sub/test_bigquery.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/cloud/events/test_replayer.py b/tests/cloud/events/test_replayer.py index f9c16b4b5..99e073dd6 100644 --- a/tests/cloud/events/test_replayer.py +++ b/tests/cloud/events/test_replayer.py @@ -51,7 +51,7 @@ def test_no_result_event(self): self.assertIsNone(result) self.assertIn("question was delivered", logging_context.output[0]) - def test_with_events(self): + def test_with_events_including_result_event(self): """Test that stored events can be replayed and the outputs extracted from the "result" event.""" with open(os.path.join(TESTS_DIR, "data", "events.json")) as f: events = json.load(f) diff --git a/tests/cloud/pub_sub/test_bigquery.py b/tests/cloud/pub_sub/test_bigquery.py index bf019bb78..0209e339a 100644 --- a/tests/cloud/pub_sub/test_bigquery.py +++ b/tests/cloud/pub_sub/test_bigquery.py @@ -4,7 +4,7 @@ from octue.cloud.pub_sub.bigquery import get_events -class TestBigQuery(TestCase): +class TestGetEvents(TestCase): def test_error_raised_if_event_kind_invalid(self): """Test that an error is raised if the event kind is invalid.""" with self.assertRaises(ValueError): From 75ca18b69d2028131b2d069c8448f5047947a6cb Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 11 Apr 2024 14:33:39 +0100 Subject: [PATCH 159/169] ENH: Issue deprecation warning if using `messages` for child emulator --- octue/cloud/emulators/child.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/octue/cloud/emulators/child.py b/octue/cloud/emulators/child.py index 5f60759b3..7614c5fd8 100644 --- a/octue/cloud/emulators/child.py +++ b/octue/cloud/emulators/child.py @@ -1,6 +1,7 @@ import copy import json import logging +import warnings from unittest.mock import patch from octue.cloud import EXCEPTIONS_MAPPING @@ -62,11 +63,23 @@ def from_file(cls, path): with open(path) as f: serialised_child_emulator = json.load(f) + if "messages" in serialised_child_emulator: + events = serialised_child_emulator["messages"] + + warnings.warn( + "Use of 'messages' as a key in an events JSON file for a child emulator is deprecated, and support for " + "it will be removed soon. Please use 'events' for the key instead.", + category=DeprecationWarning, + ) + + else: + events = serialised_child_emulator.get("events") + return cls( id=serialised_child_emulator.get("id"), backend=serialised_child_emulator.get("backend"), internal_service_name=serialised_child_emulator.get("internal_service_name"), - events=serialised_child_emulator.get("events"), + events=events, ) def __repr__(self): From 101683112d3b783fe1de21064281aeba01ee338f Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 11 Apr 2024 14:41:44 +0100 Subject: [PATCH 160/169] DOC: Document `Manifest.download` method --- docs/source/manifest.rst | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/docs/source/manifest.rst b/docs/source/manifest.rst index 69eb1b235..8d00af312 100644 --- a/docs/source/manifest.rst +++ b/docs/source/manifest.rst @@ -56,11 +56,12 @@ See :doc:`here ` for more information. Receive datasets from a service ------------------------------- -Get output datasets from an Octue service from the cloud when you're ready. +Access output datasets from an Octue service from the cloud when you're ready. .. code-block:: python - answer["output_manifest"]["an_output_dataset"].files + manifest = answer["output_manifest"] + manifest["an_output_dataset"].files >>> , })> .. hint:: @@ -69,6 +70,18 @@ Get output datasets from an Octue service from the cloud when you're ready. them - the output manifest is this reference. You’ll need to use it straight away or save it to make use of it. +Download all datasets from a manifest +------------------------------------- +Download all or a subset of datasets from a manifest. + +.. code-block:: python + + manifest.download() + >>> { + "my_dataset": "/path/to/dataset" + } + + Further information =================== From 2f36b9881030b880f881f6d5d2533da464c37db5 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 11 Apr 2024 15:03:39 +0100 Subject: [PATCH 161/169] DOC: Update `Child` docstrings --- octue/resources/child.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/octue/resources/child.py b/octue/resources/child.py index 80d6377e6..b0208472d 100644 --- a/octue/resources/child.py +++ b/octue/resources/child.py @@ -72,7 +72,7 @@ def ask( - A synchronous (ask-and-wait) question and wait for it to return an output. Questions are synchronous if the `push_endpoint` isn't provided and `asynchronous=False`. - An asynchronous (fire-and-forget) question and return immediately. To make a question asynchronous, provide - the `push_endpoint` argument or set `asynchronous=False`. + the `push_endpoint` argument or set `asynchronous=True`. :param any|None input_values: any input values for the question, conforming with the schema in the child's twine :param octue.resources.manifest.Manifest|None input_manifest: an input manifest of any datasets needed for the question, conforming with the schema in the child's twine @@ -84,7 +84,7 @@ def ask( :param str save_diagnostics: must be one of {"SAVE_DIAGNOSTICS_OFF", "SAVE_DIAGNOSTICS_ON_CRASH", "SAVE_DIAGNOSTICS_ON"}; if turned on, allow the input values and manifest (and its datasets) to be saved by the child either all the time or just if it fails while processing them :param str|None question_uuid: the UUID to use for the question if a specific one is needed; a UUID is generated if not :param str|None push_endpoint: if answers to the question should be pushed to an endpoint, provide its URL here (the returned subscription will be a push subscription); if not, leave this as `None` - :param bool asynchronous: if `True`, don't create an answer subscription + :param bool asynchronous: if `True`, don't wait for an answer or create an answer subscription (the result and other events can be retrieved from the event store later) :param float timeout: time in seconds to wait for an answer before raising a timeout error :param float|int maximum_heartbeat_interval: the maximum amount of time (in seconds) allowed between child heartbeats before an error is raised :raise TimeoutError: if the timeout is exceeded while waiting for an answer From 26532f802a56068a44c98534e25405089ecd027b Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 11 Apr 2024 15:20:46 +0100 Subject: [PATCH 162/169] DOC: Document asynchronous questions and answer retrieval --- docs/source/asking_questions.rst | 148 ++++++++++++++++++++++++++++--- 1 file changed, 138 insertions(+), 10 deletions(-) diff --git a/docs/source/asking_questions.rst b/docs/source/asking_questions.rst index 7ca817221..f249780dc 100644 --- a/docs/source/asking_questions.rst +++ b/docs/source/asking_questions.rst @@ -4,11 +4,27 @@ Asking services questions ========================= -How to ask a question -===================== +What is a question? +=================== +A question is a set of data (input values and/or an input manifest) sent to a child for processing/analysis. Questions +can be: + +- **Synchronous ("ask-and-wait"):** A question whose answer is waited for in real time + +- **Asynchronous ("fire-and-forget"):** A question whose answer is not waited for and is instead retrieved later. There + are two types: + + - Regular: responses to these questions are automatically stored in an event store where they can be :ref:`retrieved using the Octue SDK ` + + - Push endpoint: responses to these questions are pushed to an HTTP endpoint for asynchronous handling using Octue's + `django-twined `_ or custom logic in your own webserver. + Questions are always asked to a *revision* of a service. You can ask a service a question if you have its -:ref:`SRUID `, project name, and the necessary permissions. The question is formed of input values -and/or an input manifest. +:ref:`SRUID `, project name, and the necessary permissions. + + +Asking a question +================= .. code-block:: python @@ -47,18 +63,130 @@ You can also set the following options when you call :mod:`Child.ask `) - ``timeout`` - how long in seconds to wait for an answer (``None`` by default - i.e. don't time out) +Exceptions raised by a child +---------------------------- If a child raises an exception while processing your question, the exception will always be forwarded and re-raised in your local service or python session. You can handle exceptions in whatever way you like. -If setting a timeout, bear in mind that the question has to reach the child, the child has to run its analysis on -the inputs sent to it (this most likely corresponds to the dominant part of the wait time), and the answer has to be -sent back to the parent. If you're not sure how long a particular analysis might take, it's best to set the timeout to -``None`` initially or ask the owner/maintainer of the child for an estimate. +Timeouts +-------- +If setting a timeout, bear in mind that the question has to reach the child, the child has to run its analysis on the +inputs sent to it (this will most likely make up the dominant part of the wait time), and the answer has to be sent back +to the parent. If you're not sure how long a particular analysis might take, it's best to set the timeout to ``None`` +initially or ask the owner/maintainer of the child for an estimate. + + +.. _retrieving_asynchronous_answers: + +Retrieving answers to asynchronous questions +============================================ +To retrieve results and other events from the processing of a question later, make sure you have the permissions to +access to event store and run: + +.. code-block:: python + + from octue.cloud.pub_sub.bigquery import get_events + + events = get_events( + table_id="your-project.your-dataset.your-table", + sender="octue/test-service:1.0.0", + question_uuid="53353901-0b47-44e7-9da3-a3ed59990a71", + ) + + +.. collapse:: See some example events here... + + .. code-block:: python + >>> events + [ + { + "event": { + "datetime": "2024-03-06T15:44:18.156044", + "kind": "delivery_acknowledgement" + }, + }, + { + "event": { + "kind": "log_record", + "log_record": { + "args": null, + "created": 1709739861.5949728, + "exc_info": null, + "exc_text": null, + "filename": "app.py", + "funcName": "run", + "levelname": "INFO", + "levelno": 20, + "lineno": 28, + "module": "app", + "msecs": 594.9728488922119, + "msg": "Finished example analysis.", + "name": "app", + "pathname": "/workspace/example_service_cloud_run/app.py", + "process": 2, + "processName": "MainProcess", + "relativeCreated": 8560.13798713684, + "stack_info": null, + "thread": 68328473233152, + "threadName": "ThreadPoolExecutor-0_2" + } + }, + }, + { + "event": { + "datetime": "2024-03-06T15:46:18.167424", + "kind": "heartbeat" + }, + "attributes": { + "datetime": "2024-04-11T10:46:48.236064", + "uuid": "a9de11b1-e88f-43fa-b3a4-40a590c3443f", + "order": "7", + "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", + "originator": "octue/test-service:1.0.0", + "sender": "octue/test-service:1.0.0", + "sender_type": "CHILD", + "sender_sdk_version": "0.51.0", + "recipient": "octue/another-service:3.2.1" + } + } + { + "event": { + "kind": "result", + "output_manifest": { + "datasets": { + "example_dataset": { + "files": [ + "gs://octue-sdk-python-test-bucket/example_output_datasets/example_dataset/output.dat" + ], + "id": "419bff6b-08c3-4c16-9eb1-5d1709168003", + "labels": [], + "name": "divergent-strange-gharial-of-pizza", + "path": "https://storage.googleapis.com/octue-sdk-python-test-bucket/example_output_datasets/example_dataset/.signed_metadata_files/divergent-strange-gharial-of-pizza", + "tags": {} + } + }, + "id": "a13713ae-f207-41c6-9e29-0a848ced6039", + "name": null + }, + "output_values": [1, 2, 3, 4, 5] + }, + }, + ] + + +**Options** + +- ``kind`` - Only retrieve this kind of event if present (e.g. "result") +- ``include_attributes`` - If ``True``, retrieve all the events' attributes as well +- ``include_backend_metadata`` - If ``True``, retrieve information about the service backend that produced the event +- ``limit`` - If set to a positive integer, limit the number of events returned to this Asking multiple questions in parallel @@ -81,7 +209,7 @@ raised and no answers are returned. This method uses multithreading, allowing all the questions to be asked at once instead of one after another. -Options: +**Options** - If ``raise_errors=False`` is provided, answers are returned for all successful questions while unraised errors are returned for unsuccessful ones From 933b4ec36b42edce4f297e7751cfa784160f2b8b Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 11 Apr 2024 15:21:25 +0100 Subject: [PATCH 163/169] DOC: Update copyright year in `conf.py` --- docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 6003967a8..023694a62 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -51,7 +51,7 @@ # General information about the project. project = "Octue SDK (Python)" author = "Octue Ltd" -copyright = "2022, Octue Ltd" +copyright = "2024, Octue Ltd" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the From 7ff11b8ab6c8eb78bdeae694ab17f401cb5549d9 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 11 Apr 2024 15:23:31 +0100 Subject: [PATCH 164/169] DOC: Fix docs typos --- docs/source/asking_questions.rst | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/docs/source/asking_questions.rst b/docs/source/asking_questions.rst index f249780dc..acc4b6448 100644 --- a/docs/source/asking_questions.rst +++ b/docs/source/asking_questions.rst @@ -14,9 +14,9 @@ can be: - **Asynchronous ("fire-and-forget"):** A question whose answer is not waited for and is instead retrieved later. There are two types: - - Regular: responses to these questions are automatically stored in an event store where they can be :ref:`retrieved using the Octue SDK ` + - **Regular:** Responses to these questions are automatically stored in an event store where they can be :ref:`retrieved using the Octue SDK ` - - Push endpoint: responses to these questions are pushed to an HTTP endpoint for asynchronous handling using Octue's + - **Push endpoint:** Responses to these questions are pushed to an HTTP endpoint for asynchronous handling using Octue's `django-twined `_ or custom logic in your own webserver. Questions are always asked to a *revision* of a service. You can ask a service a question if you have its @@ -88,7 +88,7 @@ initially or ask the owner/maintainer of the child for an estimate. Retrieving answers to asynchronous questions ============================================ To retrieve results and other events from the processing of a question later, make sure you have the permissions to -access to event store and run: +access the event store and run: .. code-block:: python @@ -101,9 +101,18 @@ access to event store and run: ) -.. collapse:: See some example events here... +**Options** + +- ``kind`` - Only retrieve this kind of event if present (e.g. "result") +- ``include_attributes`` - If ``True``, retrieve all the events' attributes as well +- ``include_backend_metadata`` - If ``True``, retrieve information about the service backend that produced the event +- ``limit`` - If set to a positive integer, limit the number of events returned to this + + +.. collapse:: See an example output here... .. code-block:: python + >>> events [ { @@ -180,14 +189,7 @@ access to event store and run: }, ] - -**Options** - -- ``kind`` - Only retrieve this kind of event if present (e.g. "result") -- ``include_attributes`` - If ``True``, retrieve all the events' attributes as well -- ``include_backend_metadata`` - If ``True``, retrieve information about the service backend that produced the event -- ``limit`` - If set to a positive integer, limit the number of events returned to this - +---- Asking multiple questions in parallel ===================================== From c83303f2301ad48530adfbcf5c042d75600ed5a3 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 11 Apr 2024 16:46:43 +0100 Subject: [PATCH 165/169] DOC: Update child emulator documentation --- docs/source/testing_services.rst | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/source/testing_services.rst b/docs/source/testing_services.rst index 7549a4c68..df2419124 100644 --- a/docs/source/testing_services.rst +++ b/docs/source/testing_services.rst @@ -22,18 +22,18 @@ your tests: The Child Emulator ------------------ -We've written a child emulator that takes a list of messages and returns them to the parent for handling in the order -given - without contacting the real child or using Pub/Sub. Any messages a real child can produce are supported. +We've written a child emulator that takes a list of events and returns them to the parent for handling in the order +given - without contacting the real child or using Pub/Sub. Any events a real child can produce are supported. :mod:`Child ` instances can be mocked like-for-like by :mod:`ChildEmulator ` instances without the parent knowing. You can provide -the emulated messages in python or via a JSON file. +the emulated events in python or via a JSON file. Message types ------------- You can emulate any message type that your app (the parent) can handle. The table below shows what these are. +-----------------------+--------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+ -| Message type | Number of messages supported | Example | +| Message type | Number of events supported | Example | +=======================+==================================================================================================+===========================================================================================================================+ | ``log_record`` | Any number | {"type": "log_record": "log_record": {"msg": "Starting analysis."}} | +-----------------------+--------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+ @@ -50,7 +50,7 @@ You can emulate any message type that your app (the parent) can handle. The tabl - The ``log_record`` key of a ``log_record`` message is any dictionary that the ``logging.makeLogRecord`` function can convert into a log record. - The ``data`` key of a ``monitor_message`` message must be a JSON-serialised string -- Any messages after a ``result`` or ``exception`` message won't be passed to the parent because execution of the child +- Any events after a ``result`` or ``exception`` message won't be passed to the parent because execution of the child emulator will have ended. @@ -59,7 +59,7 @@ Instantiating a child emulator in python .. code-block:: python - messages = [ + events = [ { "type": "log_record", "log_record": {"msg": "Starting analysis."}, @@ -81,7 +81,7 @@ Instantiating a child emulator in python child_emulator = ChildEmulator( backend={"name": "GCPPubSubBackend", "project_name": "my-project"}, - messages=messages + events=events ) def handle_monitor_message(message): @@ -96,14 +96,14 @@ Instantiating a child emulator in python Instantiating a child emulator from a JSON file ----------------------------------------------- -You can provide a JSON file with either just messages in or with messages and some or all of the +You can provide a JSON file with either just events in or with events and some or all of the :mod:`ChildEmulator ` constructor parameters. Here's an example JSON file -with just the messages: +with just the events: .. code-block:: json { - "messages": [ + "events": [ { "type": "log_record", "log_record": {"msg": "Starting analysis."} @@ -179,7 +179,7 @@ To emulate your children in tests, patch the :mod:`Child >> [ { 'type': 'delivery_acknowledgement', @@ -257,7 +257,7 @@ You can then feed these into a child emulator to emulate one possible response o from octue.cloud.emulators import ChildEmulator - child_emulator = ChildEmulator(messages=child.received_messages) + child_emulator = ChildEmulator(events=child.received_events) child_emulator.ask(input_values=[1, 2, 3, 4]) >>> {"some": "results"} From 14481ace81c33479960a9b1a280204b8abd9b083 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 11 Apr 2024 16:46:59 +0100 Subject: [PATCH 166/169] OPS: Update codecov GitHub actions and provide token --- .github/workflows/python-ci.yml | 3 ++- .github/workflows/release.yml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 10911b146..d830f7187 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -63,10 +63,11 @@ jobs: run: tox -vv -e py - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: files: coverage.xml fail_ci_if_error: true + token: ${{ secrets.CODECOV_TOKEN }} test-publish: if: "!contains(github.event.head_commit.message, 'skipci')" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 38b4f8d35..eb6b95e38 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -59,10 +59,11 @@ jobs: run: tox -vv -e py - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: files: coverage.xml fail_ci_if_error: false + token: ${{ secrets.CODECOV_TOKEN }} outputs: package_version: ${{ steps.get-package-version.outputs.PACKAGE_VERSION }} From 9ae506e8f740416a874f57263875214083312684 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 11 Apr 2024 17:07:38 +0100 Subject: [PATCH 167/169] CHO: Add version compatibility data --- octue/metadata/recorded_questions.jsonl | 1 + octue/metadata/version_compatibilities.json | 183 +++++++++++++++----- 2 files changed, 139 insertions(+), 45 deletions(-) diff --git a/octue/metadata/recorded_questions.jsonl b/octue/metadata/recorded_questions.jsonl index ee9e31f53..ebc03df02 100644 --- a/octue/metadata/recorded_questions.jsonl +++ b/octue/metadata/recorded_questions.jsonl @@ -43,3 +43,4 @@ {"parent_sdk_version": "0.51.0", "question": {"data": "{\"kind\": \"question\", \"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": {\"id\": \"3bf3b178-3cf1-49aa-a1d9-89cd8ec05b1a\", \"name\": null, \"datasets\": {\"my_dataset\": \"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmppu85nw5c\"}}}", "attributes": {"question_uuid": "6f7f6e1c-7632-4b4d-b91d-6f58dcb43c40", "sender_type": "PARENT", "forward_logs": "1", "save_diagnostics": "SAVE_DIAGNOSTICS_ON_CRASH", "version": "0.51.0", "message_number": "0"}}} {"parent_sdk_version": "0.52.0", "question": {"data": "{\"kind\": \"question\", \"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": {\"id\": \"6be875b3-33d8-4b00-b7ea-553f8f69bce5\", \"name\": null, \"datasets\": {\"my_dataset\": \"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmphypwu9uh\"}}}", "attributes": {"question_uuid": "cd0a78be-fda2-4730-bdba-ba6b04e4787f", "sender_type": "PARENT", "forward_logs": "1", "save_diagnostics": "SAVE_DIAGNOSTICS_ON_CRASH", "version": "0.52.0", "message_number": "0"}}} {"parent_sdk_version": "0.52.1", "question": {"data": "{\"kind\": \"question\", \"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": {\"id\": \"6be875b3-33d8-4b00-b7ea-553f8f69bce5\", \"name\": null, \"datasets\": {\"my_dataset\": \"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmphypwu9uh\"}}}", "attributes": {"question_uuid": "cd0a78be-fda2-4730-bdba-ba6b04e4787f", "sender_type": "PARENT", "forward_logs": "1", "save_diagnostics": "SAVE_DIAGNOSTICS_ON_CRASH", "version": "0.52.1", "message_number": "0"}}} +{"parent_sdk_version": "0.53.0", "question": {"data": "{\"kind\": \"question\", \"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": {\"id\": \"6be875b3-33d8-4b00-b7ea-553f8f69bce5\", \"name\": null, \"datasets\": {\"my_dataset\": \"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmphypwu9uh\"}}}", "attributes": {"datetime": "2024-04-11T10:46:48.236064", "uuid": "a9de11b1-e88f-43fa-b3a4-40a590c3443f", "question_uuid": "cd0a78be-fda2-4730-bdba-ba6b04e4787f", "forward_logs": "1", "save_diagnostics": "SAVE_DIAGNOSTICS_ON_CRASH", "originator": "octue/test-service:0.1.0", "sender": "octue/test-service:0.1.0", "sender_type": "PARENT", "sender_sdk_version": "0.53.0", "recipient": "octue/another-service:1.0.12", "order": "0"}}} diff --git a/octue/metadata/version_compatibilities.json b/octue/metadata/version_compatibilities.json index f18c95549..e1403dd15 100644 --- a/octue/metadata/version_compatibilities.json +++ b/octue/metadata/version_compatibilities.json @@ -44,7 +44,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.34.1": { "0.40.1": true, @@ -91,7 +92,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.34.0": { "0.40.1": true, @@ -138,7 +140,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.33.0": { "0.40.1": true, @@ -185,7 +188,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.32.0": { "0.40.1": true, @@ -232,7 +236,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.31.0": { "0.40.1": true, @@ -279,7 +284,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.30.0": { "0.40.1": true, @@ -326,7 +332,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.36.0": { "0.40.1": true, @@ -373,7 +380,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.37.0": { "0.40.1": true, @@ -420,7 +428,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.38.0": { "0.40.1": true, @@ -467,7 +476,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.38.1": { "0.40.1": true, @@ -514,7 +524,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.39.0": { "0.40.1": true, @@ -561,7 +572,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.40.0": { "0.40.1": true, @@ -608,7 +620,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.40.1": { "0.40.1": true, @@ -655,7 +668,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.40.2": { "0.41.0": true, @@ -702,7 +716,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.41.0": { "0.41.0": true, @@ -749,7 +764,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.41.1": { "0.41.1": true, @@ -796,7 +812,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.42.0": { "0.42.0": true, @@ -843,7 +860,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.42.1": { "0.43.2": true, @@ -890,7 +908,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.43.0": { "0.43.2": true, @@ -937,7 +956,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.43.1": { "0.43.2": true, @@ -984,7 +1004,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.43.2": { "0.43.2": true, @@ -1031,7 +1052,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.43.3": { "0.43.3": true, @@ -1078,7 +1100,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.43.4": { "0.43.4": true, @@ -1125,7 +1148,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.43.5": { "0.43.5": true, @@ -1172,7 +1196,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.43.6": { "0.43.6": true, @@ -1219,7 +1244,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.43.7": { "0.43.7": true, @@ -1266,7 +1292,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.44.0": { "0.44.0": true, @@ -1313,7 +1340,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.45.0": { "0.45.0": true, @@ -1360,7 +1388,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.46.0": { "0.46.0": true, @@ -1407,7 +1436,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.46.1": { "0.46.1": true, @@ -1454,7 +1484,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.46.2": { "0.46.2": true, @@ -1501,7 +1532,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.46.3": { "0.46.3": true, @@ -1548,7 +1580,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.47.0": { "0.47.0": true, @@ -1595,7 +1628,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.47.1": { "0.47.1": true, @@ -1642,7 +1676,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.47.2": { "0.47.2": true, @@ -1689,7 +1724,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.48.0": { "0.48.0": true, @@ -1736,7 +1772,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.49.0": { "0.49.1": true, @@ -1783,7 +1820,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.49.1": { "0.49.1": true, @@ -1830,7 +1868,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.49.2": { "0.49.2": true, @@ -1877,7 +1916,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.50.0": { "0.50.0": true, @@ -1924,7 +1964,8 @@ "0.31.0": true, "0.30.0": true, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.50.1": { "0.51.0": false, @@ -1971,7 +2012,8 @@ "0.31.0": true, "0.30.0": true, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.51.0": { "0.51.0": true, @@ -2018,7 +2060,8 @@ "0.31.0": false, "0.30.0": false, "0.52.0": true, - "0.52.1": true + "0.52.1": true, + "0.53.0": false }, "0.52.0": { "0.51.0": true, @@ -2065,7 +2108,8 @@ "0.31.0": false, "0.30.0": false, "0.52.0": true, - "0.52.1": true + "0.52.1": true, + "0.53.0": false }, "0.52.1": { "0.51.0": true, @@ -2112,6 +2156,55 @@ "0.31.0": false, "0.30.0": false, "0.52.0": true, - "0.52.1": true + "0.52.1": true, + "0.53.0": false + }, + "0.53.0": { + "0.51.0": false, + "0.50.1": false, + "0.50.0": false, + "0.49.2": false, + "0.49.1": false, + "0.49.0": false, + "0.48.0": false, + "0.47.2": false, + "0.47.1": false, + "0.47.0": false, + "0.46.3": false, + "0.46.2": false, + "0.46.1": false, + "0.46.0": false, + "0.45.0": false, + "0.44.0": false, + "0.43.7": false, + "0.43.6": false, + "0.43.5": false, + "0.43.4": false, + "0.43.3": false, + "0.43.2": false, + "0.43.1": false, + "0.43.0": false, + "0.42.1": false, + "0.42.0": false, + "0.41.1": false, + "0.41.0": false, + "0.40.2": false, + "0.40.1": false, + "0.40.0": false, + "0.39.0": false, + "0.38.1": false, + "0.38.0": false, + "0.37.0": false, + "0.36.0": false, + "0.35.0": false, + "0.34.1": false, + "0.34.0": false, + "0.33.0": false, + "0.32.0": false, + "0.31.0": false, + "0.30.0": false, + "0.52.0": false, + "0.52.1": false, + "0.53.0": true } } From a759b4ceedc946f4b1e87740f8efe7edda3d3d5a Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 11 Apr 2024 17:16:08 +0100 Subject: [PATCH 168/169] DOC: Remove version compatbility information for versions `<0.40.0` --- octue/metadata/recorded_questions.jsonl | 12 - octue/metadata/version_compatibilities.json | 984 -------------------- 2 files changed, 996 deletions(-) diff --git a/octue/metadata/recorded_questions.jsonl b/octue/metadata/recorded_questions.jsonl index ebc03df02..c658619e1 100644 --- a/octue/metadata/recorded_questions.jsonl +++ b/octue/metadata/recorded_questions.jsonl @@ -1,15 +1,3 @@ -{"parent_sdk_version": "0.30.0", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": \"{\\n \\\"datasets\\\": {\\n \\\"my_dataset\\\": \\\"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmpw94cjhfq\\\"\\n },\\n \\\"id\\\": \\\"cf38d606-c9d2-4f54-a806-6337b1fa30fb\\\",\\n \\\"name\\\": null\\n}\"}", "attributes": {"question_uuid": "8157304e-e056-4ec9-80ef-dce540c27191", "forward_logs": "1", "octue_sdk_version": "0.30.0"}}} -{"parent_sdk_version": "0.31.0", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": \"{\\n \\\"datasets\\\": {\\n \\\"my_dataset\\\": \\\"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmpzrl_zrel\\\"\\n },\\n \\\"id\\\": \\\"b2d0a0cf-cdc6-48d8-92a7-d7cbee14807f\\\",\\n \\\"name\\\": null\\n}\"}", "attributes": {"question_uuid": "130b92bb-4486-4785-92ab-96f641367b69", "forward_logs": "1", "octue_sdk_version": "0.31.0", "allow_save_diagnostics_data_on_crash": "1"}}} -{"parent_sdk_version": "0.32.0", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": \"{\\n \\\"datasets\\\": {\\n \\\"my_dataset\\\": \\\"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmpg9dgmfjy\\\"\\n },\\n \\\"id\\\": \\\"7f33f9f5-79e6-46af-b450-50c1b7ad19d5\\\",\\n \\\"name\\\": null\\n}\"}", "attributes": {"question_uuid": "9e982e65-5937-4e89-9c69-d86640805ce4", "forward_logs": "1", "octue_sdk_version": "0.32.0", "allow_save_diagnostics_data_on_crash": "1"}}} -{"parent_sdk_version": "0.33.0", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": \"{\\n \\\"datasets\\\": {\\n \\\"my_dataset\\\": \\\"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmpeofjz_r9\\\"\\n },\\n \\\"id\\\": \\\"8b860309-4c9f-43a0-bf3a-621a59ad02cd\\\",\\n \\\"name\\\": null\\n}\"}", "attributes": {"question_uuid": "4414295f-fd07-4447-9170-fae08e525614", "forward_logs": "1", "octue_sdk_version": "0.33.0", "allow_save_diagnostics_data_on_crash": "1"}}} -{"parent_sdk_version": "0.34.0", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": \"{\\n \\\"datasets\\\": {\\n \\\"my_dataset\\\": \\\"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmp8enl0nnh\\\"\\n },\\n \\\"id\\\": \\\"0b602f70-741c-4ec7-bde4-d6e18ba5213f\\\",\\n \\\"name\\\": null\\n}\"}", "attributes": {"question_uuid": "8b0d7b63-ee2a-4d3b-8cfd-ab98fc389c78", "forward_logs": "1", "allow_save_diagnostics_data_on_crash": "1", "octue_sdk_version": "0.34.0"}}} -{"parent_sdk_version": "0.34.1", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": \"{\\n \\\"datasets\\\": {\\n \\\"my_dataset\\\": \\\"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmp1y2hp_71\\\"\\n },\\n \\\"id\\\": \\\"defb3d77-1d60-48e3-9270-4364badccb3d\\\",\\n \\\"name\\\": null\\n}\"}", "attributes": {"question_uuid": "15cba19c-401a-4321-9f73-67c78f28016e", "forward_logs": "1", "allow_save_diagnostics_data_on_crash": "1", "octue_sdk_version": "0.34.1"}}} -{"parent_sdk_version": "0.35.0", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": \"{\\n \\\"datasets\\\": {\\n \\\"my_dataset\\\": \\\"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmpsx4bxiyq\\\"\\n },\\n \\\"id\\\": \\\"56538ac2-7e4b-4f8b-8d03-bfba60881cea\\\",\\n \\\"name\\\": null\\n}\"}", "attributes": {"question_uuid": "f8f30ded-714f-4719-9622-61e850ceb872", "forward_logs": "1", "allow_save_diagnostics_data_on_crash": "1", "octue_sdk_version": "0.35.0"}}} -{"parent_sdk_version": "0.36.0", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": \"{\\n \\\"datasets\\\": {\\n \\\"my_dataset\\\": \\\"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmpj6b99swv\\\"\\n },\\n \\\"id\\\": \\\"1578e98c-a449-42e1-99c3-9f16eb64e04e\\\",\\n \\\"name\\\": null\\n}\"}", "attributes": {"question_uuid": "8fc1eecd-e6f4-4985-8a2f-9873f1401eed", "forward_logs": "1", "allow_save_diagnostics_data_on_crash": "1", "octue_sdk_version": "0.36.0"}}} -{"parent_sdk_version": "0.37.0", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": \"{\\n \\\"datasets\\\": {\\n \\\"my_dataset\\\": \\\"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmp4394fsdk\\\"\\n },\\n \\\"id\\\": \\\"98fe6a81-e6d0-4e1b-bb70-2d7490024e3e\\\",\\n \\\"name\\\": null\\n}\"}", "attributes": {"question_uuid": "07d706d9-5df3-44a9-a133-fc56c235559a", "forward_logs": "1", "allow_save_diagnostics_data_on_crash": "1", "octue_sdk_version": "0.37.0"}}} -{"parent_sdk_version": "0.38.0", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": \"{\\n \\\"datasets\\\": {\\n \\\"my_dataset\\\": \\\"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmpc_nmpiz2\\\"\\n },\\n \\\"id\\\": \\\"739b1ae5-99f8-4bd0-b344-38f35d74b8a8\\\",\\n \\\"name\\\": null\\n}\"}", "attributes": {"question_uuid": "7bb5e29f-c8de-4fb4-923b-b5c854a24f49", "forward_logs": "1", "allow_save_diagnostics_data_on_crash": "1", "octue_sdk_version": "0.38.0"}}} -{"parent_sdk_version": "0.38.1", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": \"{\\n \\\"datasets\\\": {\\n \\\"my_dataset\\\": \\\"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmpt1uu7685\\\"\\n },\\n \\\"id\\\": \\\"8b3c39f9-5232-4898-998d-934defdc5626\\\",\\n \\\"name\\\": null\\n}\"}", "attributes": {"question_uuid": "a96e6ec5-3798-4160-b45f-6f248834f740", "forward_logs": "1", "allow_save_diagnostics_data_on_crash": "1", "octue_sdk_version": "0.38.1"}}} -{"parent_sdk_version": "0.39.0", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": \"{\\n \\\"datasets\\\": {\\n \\\"my_dataset\\\": \\\"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmp664lr951\\\"\\n },\\n \\\"id\\\": \\\"57a78545-f40f-4971-bff9-b138bf1d6292\\\",\\n \\\"name\\\": null\\n}\", \"children\": null}", "attributes": {"question_uuid": "7a8eed0b-bd2d-4eb7-93b3-5eb8244c812d", "forward_logs": "1", "allow_save_diagnostics_data_on_crash": "1", "octue_sdk_version": "0.39.0"}}} {"parent_sdk_version": "0.40.0", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": \"{\\n \\\"datasets\\\": {\\n \\\"my_dataset\\\": \\\"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmppnbc6rcc\\\"\\n },\\n \\\"id\\\": \\\"31440887-beef-41a3-935f-08adbe8dbbdc\\\",\\n \\\"name\\\": null\\n}\", \"children\": null}", "attributes": {"question_uuid": "f66edd6e-517c-4c26-9d60-16cb508e5fe1", "forward_logs": "1", "allow_save_diagnostics_data_on_crash": "1", "octue_sdk_version": "0.40.0"}}} {"parent_sdk_version": "0.40.1", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": \"{\\n \\\"datasets\\\": {\\n \\\"my_dataset\\\": \\\"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmppnbc6rcc\\\"\\n },\\n \\\"id\\\": \\\"31440887-beef-41a3-935f-08adbe8dbbdc\\\",\\n \\\"name\\\": null\\n}\", \"children\": null}", "attributes": {"question_uuid": "f66edd6e-517c-4c26-9d60-16cb508e5fe1", "forward_logs": "1", "allow_save_diagnostics_data_on_crash": "1", "octue_sdk_version": "0.40.1"}}} {"parent_sdk_version": "0.40.2", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": \"{\\n \\\"datasets\\\": {\\n \\\"my_dataset\\\": \\\"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmprj5sl5al\\\"\\n },\\n \\\"id\\\": \\\"6ce744fb-fce6-4346-949a-109675ca7a5a\\\",\\n \\\"name\\\": null\\n}\", \"children\": null}", "attributes": {"question_uuid": "42d9a880-8672-4be6-a3ae-b8518a9e58db", "forward_logs": "1", "allow_save_diagnostics_data_on_crash": "1", "octue_sdk_version": "0.40.2"}}} diff --git a/octue/metadata/version_compatibilities.json b/octue/metadata/version_compatibilities.json index e1403dd15..712c71f5f 100644 --- a/octue/metadata/version_compatibilities.json +++ b/octue/metadata/version_compatibilities.json @@ -1,595 +1,7 @@ { - "0.35.0": { - "0.40.1": true, - "0.38.1": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, - "0.36.0": true, - "0.37.0": true, - "0.38.0": true, - "0.39.0": true, - "0.40.0": true, - "0.40.2": true, - "0.41.0": true, - "0.41.1": true, - "0.42.0": true, - "0.42.1": true, - "0.43.0": true, - "0.43.1": true, - "0.43.2": true, - "0.43.3": true, - "0.43.4": true, - "0.43.5": true, - "0.43.6": true, - "0.43.7": true, - "0.44.0": true, - "0.45.0": true, - "0.46.0": true, - "0.46.1": true, - "0.46.2": true, - "0.46.3": true, - "0.47.0": true, - "0.47.1": true, - "0.47.2": true, - "0.48.0": true, - "0.49.0": true, - "0.49.1": true, - "0.49.2": true, - "0.50.0": true, - "0.50.1": true, - "0.51.0": false, - "0.52.0": false, - "0.52.1": false, - "0.53.0": false - }, - "0.34.1": { - "0.40.1": true, - "0.38.1": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, - "0.36.0": true, - "0.37.0": true, - "0.38.0": true, - "0.39.0": true, - "0.40.0": true, - "0.40.2": true, - "0.41.0": true, - "0.41.1": true, - "0.42.0": true, - "0.42.1": true, - "0.43.0": true, - "0.43.1": true, - "0.43.2": true, - "0.43.3": true, - "0.43.4": true, - "0.43.5": true, - "0.43.6": true, - "0.43.7": true, - "0.44.0": true, - "0.45.0": true, - "0.46.0": true, - "0.46.1": true, - "0.46.2": true, - "0.46.3": true, - "0.47.0": true, - "0.47.1": true, - "0.47.2": true, - "0.48.0": true, - "0.49.0": true, - "0.49.1": true, - "0.49.2": true, - "0.50.0": true, - "0.50.1": true, - "0.51.0": false, - "0.52.0": false, - "0.52.1": false, - "0.53.0": false - }, - "0.34.0": { - "0.40.1": true, - "0.38.1": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, - "0.36.0": true, - "0.37.0": true, - "0.38.0": true, - "0.39.0": true, - "0.40.0": true, - "0.40.2": true, - "0.41.0": true, - "0.41.1": true, - "0.42.0": true, - "0.42.1": true, - "0.43.0": true, - "0.43.1": true, - "0.43.2": true, - "0.43.3": true, - "0.43.4": true, - "0.43.5": true, - "0.43.6": true, - "0.43.7": true, - "0.44.0": true, - "0.45.0": true, - "0.46.0": true, - "0.46.1": true, - "0.46.2": true, - "0.46.3": true, - "0.47.0": true, - "0.47.1": true, - "0.47.2": true, - "0.48.0": true, - "0.49.0": true, - "0.49.1": true, - "0.49.2": true, - "0.50.0": true, - "0.50.1": true, - "0.51.0": false, - "0.52.0": false, - "0.52.1": false, - "0.53.0": false - }, - "0.33.0": { - "0.40.1": true, - "0.38.1": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, - "0.36.0": true, - "0.37.0": true, - "0.38.0": true, - "0.39.0": true, - "0.40.0": true, - "0.40.2": true, - "0.41.0": true, - "0.41.1": true, - "0.42.0": true, - "0.42.1": true, - "0.43.0": true, - "0.43.1": true, - "0.43.2": true, - "0.43.3": true, - "0.43.4": true, - "0.43.5": true, - "0.43.6": true, - "0.43.7": true, - "0.44.0": true, - "0.45.0": true, - "0.46.0": true, - "0.46.1": true, - "0.46.2": true, - "0.46.3": true, - "0.47.0": true, - "0.47.1": true, - "0.47.2": true, - "0.48.0": true, - "0.49.0": true, - "0.49.1": true, - "0.49.2": true, - "0.50.0": true, - "0.50.1": true, - "0.51.0": false, - "0.52.0": false, - "0.52.1": false, - "0.53.0": false - }, - "0.32.0": { - "0.40.1": true, - "0.38.1": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, - "0.36.0": true, - "0.37.0": true, - "0.38.0": true, - "0.39.0": true, - "0.40.0": true, - "0.40.2": true, - "0.41.0": true, - "0.41.1": true, - "0.42.0": true, - "0.42.1": true, - "0.43.0": true, - "0.43.1": true, - "0.43.2": true, - "0.43.3": true, - "0.43.4": true, - "0.43.5": true, - "0.43.6": true, - "0.43.7": true, - "0.44.0": true, - "0.45.0": true, - "0.46.0": true, - "0.46.1": true, - "0.46.2": true, - "0.46.3": true, - "0.47.0": true, - "0.47.1": true, - "0.47.2": true, - "0.48.0": true, - "0.49.0": true, - "0.49.1": true, - "0.49.2": true, - "0.50.0": true, - "0.50.1": true, - "0.51.0": false, - "0.52.0": false, - "0.52.1": false, - "0.53.0": false - }, - "0.31.0": { - "0.40.1": true, - "0.38.1": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, - "0.36.0": true, - "0.37.0": true, - "0.38.0": true, - "0.39.0": true, - "0.40.0": true, - "0.40.2": true, - "0.41.0": true, - "0.41.1": true, - "0.42.0": true, - "0.42.1": true, - "0.43.0": true, - "0.43.1": true, - "0.43.2": true, - "0.43.3": true, - "0.43.4": true, - "0.43.5": true, - "0.43.6": true, - "0.43.7": true, - "0.44.0": true, - "0.45.0": true, - "0.46.0": true, - "0.46.1": true, - "0.46.2": true, - "0.46.3": true, - "0.47.0": true, - "0.47.1": true, - "0.47.2": true, - "0.48.0": true, - "0.49.0": true, - "0.49.1": true, - "0.49.2": true, - "0.50.0": true, - "0.50.1": true, - "0.51.0": false, - "0.52.0": false, - "0.52.1": false, - "0.53.0": false - }, - "0.30.0": { - "0.40.1": true, - "0.38.1": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, - "0.36.0": true, - "0.37.0": true, - "0.38.0": true, - "0.39.0": true, - "0.40.0": true, - "0.40.2": true, - "0.41.0": true, - "0.41.1": true, - "0.42.0": true, - "0.42.1": true, - "0.43.0": true, - "0.43.1": true, - "0.43.2": true, - "0.43.3": true, - "0.43.4": true, - "0.43.5": true, - "0.43.6": true, - "0.43.7": true, - "0.44.0": true, - "0.45.0": true, - "0.46.0": true, - "0.46.1": true, - "0.46.2": true, - "0.46.3": true, - "0.47.0": true, - "0.47.1": true, - "0.47.2": true, - "0.48.0": true, - "0.49.0": true, - "0.49.1": true, - "0.49.2": true, - "0.50.0": true, - "0.50.1": true, - "0.51.0": false, - "0.52.0": false, - "0.52.1": false, - "0.53.0": false - }, - "0.36.0": { - "0.40.1": true, - "0.38.1": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, - "0.37.0": true, - "0.38.0": true, - "0.39.0": true, - "0.40.0": true, - "0.40.2": true, - "0.41.0": true, - "0.41.1": true, - "0.42.0": true, - "0.42.1": true, - "0.43.0": true, - "0.43.1": true, - "0.43.2": true, - "0.43.3": true, - "0.43.4": true, - "0.43.5": true, - "0.43.6": true, - "0.43.7": true, - "0.44.0": true, - "0.45.0": true, - "0.46.0": true, - "0.46.1": true, - "0.46.2": true, - "0.46.3": true, - "0.47.0": true, - "0.47.1": true, - "0.47.2": true, - "0.48.0": true, - "0.49.0": true, - "0.49.1": true, - "0.49.2": true, - "0.50.0": true, - "0.50.1": true, - "0.51.0": false, - "0.52.0": false, - "0.52.1": false, - "0.53.0": false - }, - "0.37.0": { - "0.40.1": true, - "0.38.1": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, - "0.38.0": true, - "0.39.0": true, - "0.40.0": true, - "0.40.2": true, - "0.41.0": true, - "0.41.1": true, - "0.42.0": true, - "0.42.1": true, - "0.43.0": true, - "0.43.1": true, - "0.43.2": true, - "0.43.3": true, - "0.43.4": true, - "0.43.5": true, - "0.43.6": true, - "0.43.7": true, - "0.44.0": true, - "0.45.0": true, - "0.46.0": true, - "0.46.1": true, - "0.46.2": true, - "0.46.3": true, - "0.47.0": true, - "0.47.1": true, - "0.47.2": true, - "0.48.0": true, - "0.49.0": true, - "0.49.1": true, - "0.49.2": true, - "0.50.0": true, - "0.50.1": true, - "0.51.0": false, - "0.52.0": false, - "0.52.1": false, - "0.53.0": false - }, - "0.38.0": { - "0.40.1": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, - "0.39.0": true, - "0.40.0": true, - "0.40.2": true, - "0.41.0": true, - "0.41.1": true, - "0.42.0": true, - "0.42.1": true, - "0.43.0": true, - "0.43.1": true, - "0.43.2": true, - "0.43.3": true, - "0.43.4": true, - "0.43.5": true, - "0.43.6": true, - "0.43.7": true, - "0.44.0": true, - "0.45.0": true, - "0.46.0": true, - "0.46.1": true, - "0.46.2": true, - "0.46.3": true, - "0.47.0": true, - "0.47.1": true, - "0.47.2": true, - "0.48.0": true, - "0.49.0": true, - "0.49.1": true, - "0.49.2": true, - "0.50.0": true, - "0.50.1": true, - "0.51.0": false, - "0.52.0": false, - "0.52.1": false, - "0.53.0": false - }, - "0.38.1": { - "0.40.1": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, - "0.39.0": true, - "0.40.0": true, - "0.40.2": true, - "0.41.0": true, - "0.41.1": true, - "0.42.0": true, - "0.42.1": true, - "0.43.0": true, - "0.43.1": true, - "0.43.2": true, - "0.43.3": true, - "0.43.4": true, - "0.43.5": true, - "0.43.6": true, - "0.43.7": true, - "0.44.0": true, - "0.45.0": true, - "0.46.0": true, - "0.46.1": true, - "0.46.2": true, - "0.46.3": true, - "0.47.0": true, - "0.47.1": true, - "0.47.2": true, - "0.48.0": true, - "0.49.0": true, - "0.49.1": true, - "0.49.2": true, - "0.50.0": true, - "0.50.1": true, - "0.51.0": false, - "0.52.0": false, - "0.52.1": false, - "0.53.0": false - }, - "0.39.0": { - "0.40.1": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, - "0.40.0": true, - "0.40.2": true, - "0.41.0": true, - "0.41.1": true, - "0.42.0": true, - "0.42.1": true, - "0.43.0": true, - "0.43.1": true, - "0.43.2": true, - "0.43.3": true, - "0.43.4": true, - "0.43.5": true, - "0.43.6": true, - "0.43.7": true, - "0.44.0": true, - "0.45.0": true, - "0.46.0": true, - "0.46.1": true, - "0.46.2": true, - "0.46.3": true, - "0.47.0": true, - "0.47.1": true, - "0.47.2": true, - "0.48.0": true, - "0.49.0": true, - "0.49.1": true, - "0.49.2": true, - "0.50.0": true, - "0.50.1": true, - "0.51.0": false, - "0.52.0": false, - "0.52.1": false, - "0.53.0": false - }, "0.40.0": { "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.40.2": true, "0.41.0": true, "0.41.1": true, @@ -626,18 +38,6 @@ "0.40.1": { "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.40.2": true, "0.41.0": true, "0.41.1": true, @@ -676,18 +76,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.41.1": true, "0.42.0": true, "0.42.1": true, @@ -724,18 +112,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.41.1": true, "0.42.0": true, "0.42.1": true, @@ -773,18 +149,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.42.0": true, "0.42.1": true, "0.43.0": true, @@ -822,18 +186,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.42.1": true, "0.43.0": true, "0.43.1": true, @@ -874,18 +226,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.43.3": true, "0.43.4": true, "0.43.5": true, @@ -922,18 +262,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.43.3": true, "0.43.4": true, "0.43.5": true, @@ -970,18 +298,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.43.3": true, "0.43.4": true, "0.43.5": true, @@ -1018,18 +334,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.43.3": true, "0.43.4": true, "0.43.5": true, @@ -1067,18 +371,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.43.4": true, "0.43.5": true, "0.43.6": true, @@ -1116,18 +408,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.43.5": true, "0.43.6": true, "0.43.7": true, @@ -1165,18 +445,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.43.6": true, "0.43.7": true, "0.44.0": true, @@ -1214,18 +482,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.43.7": true, "0.44.0": true, "0.45.0": true, @@ -1263,18 +519,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.44.0": true, "0.45.0": true, "0.46.0": true, @@ -1312,18 +556,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.45.0": true, "0.46.0": true, "0.46.1": true, @@ -1361,18 +593,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.46.0": true, "0.46.1": true, "0.46.2": true, @@ -1410,18 +630,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.46.1": true, "0.46.2": true, "0.46.3": true, @@ -1459,18 +667,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.46.2": true, "0.46.3": true, "0.47.0": true, @@ -1508,18 +704,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.46.3": true, "0.47.0": true, "0.47.1": true, @@ -1557,18 +741,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.47.0": true, "0.47.1": true, "0.47.2": true, @@ -1606,18 +778,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.47.1": true, "0.47.2": true, "0.48.0": true, @@ -1655,18 +815,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.47.2": true, "0.48.0": true, "0.49.0": true, @@ -1704,18 +852,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.48.0": true, "0.49.0": true, "0.49.1": true, @@ -1753,18 +889,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.49.0": true, "0.49.1": true, "0.49.2": true, @@ -1803,18 +927,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.49.2": true, "0.50.0": true, "0.50.1": true, @@ -1851,18 +963,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.49.2": true, "0.50.0": true, "0.50.1": true, @@ -1900,18 +1000,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.50.0": true, "0.50.1": true, "0.51.0": false, @@ -1951,18 +1039,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.52.0": false, "0.52.1": false, "0.53.0": false @@ -1999,18 +1075,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.52.0": false, "0.52.1": false, "0.53.0": false @@ -2047,18 +1111,6 @@ "0.40.2": false, "0.40.1": false, "0.40.0": false, - "0.39.0": false, - "0.38.1": false, - "0.38.0": false, - "0.37.0": false, - "0.36.0": false, - "0.35.0": false, - "0.34.1": false, - "0.34.0": false, - "0.33.0": false, - "0.32.0": false, - "0.31.0": false, - "0.30.0": false, "0.52.0": true, "0.52.1": true, "0.53.0": false @@ -2095,18 +1147,6 @@ "0.40.2": false, "0.40.1": false, "0.40.0": false, - "0.39.0": false, - "0.38.1": false, - "0.38.0": false, - "0.37.0": false, - "0.36.0": false, - "0.35.0": false, - "0.34.1": false, - "0.34.0": false, - "0.33.0": false, - "0.32.0": false, - "0.31.0": false, - "0.30.0": false, "0.52.0": true, "0.52.1": true, "0.53.0": false @@ -2143,18 +1183,6 @@ "0.40.2": false, "0.40.1": false, "0.40.0": false, - "0.39.0": false, - "0.38.1": false, - "0.38.0": false, - "0.37.0": false, - "0.36.0": false, - "0.35.0": false, - "0.34.1": false, - "0.34.0": false, - "0.33.0": false, - "0.32.0": false, - "0.31.0": false, - "0.30.0": false, "0.52.0": true, "0.52.1": true, "0.53.0": false @@ -2191,18 +1219,6 @@ "0.40.2": false, "0.40.1": false, "0.40.0": false, - "0.39.0": false, - "0.38.1": false, - "0.38.0": false, - "0.37.0": false, - "0.36.0": false, - "0.35.0": false, - "0.34.1": false, - "0.34.0": false, - "0.33.0": false, - "0.32.0": false, - "0.31.0": false, - "0.30.0": false, "0.52.0": false, "0.52.1": false, "0.53.0": true From cbe5eb692790b080411313213533e3e82f9587ea Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 11 Apr 2024 17:16:36 +0100 Subject: [PATCH 169/169] DOC: Update inter-service compatibility doc skipci --- docs/source/inter_service_compatibility.rst | 176 +++++++++----------- 1 file changed, 78 insertions(+), 98 deletions(-) diff --git a/docs/source/inter_service_compatibility.rst b/docs/source/inter_service_compatibility.rst index 86016b351..492dedff9 100644 --- a/docs/source/inter_service_compatibility.rst +++ b/docs/source/inter_service_compatibility.rst @@ -5,107 +5,87 @@ Inter-service compatibility Octue services acting as parents and children communicate with each other according to the `services communication schema `_. Up until version ``0.51.0``, services running nearly all versions of ``octue`` could communicate with each other compatibly. To allow a significant infrastructure upgrade, -version ``0.51.0`` introduced a number of breaking changes to the standard meaning services running ``0.51.0`` or -greater are only able to communicate with other services running ``0.51.0`` or greater. The table below shows which -``octue`` versions parents can run (rows) to send questions compatible with versions children are running (columns). -Note that this table does not display whether children's responses are compatible with the parent, just that a child is -able to accept a question. +version ``0.51.0`` introduced a number of breaking changes to the standard meaning services running versions ``0.51.0`` +to ``0.52.1`` are only able to communicate with other services running versions in the same range. The same applies to +services running versions ``>=0.53.0``. + +The table below shows which ``octue`` versions parents can run (rows) to send questions compatible with versions +children are running (columns). Note that this table does not display whether children's responses are compatible with +the parent, just that a child is able to accept a question. **Key** - ``0`` = incompatible - ``1`` = compatible -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| | 0.52.1 | 0.52.0 | 0.51.0 | 0.50.1 | 0.50.0 | 0.49.2 | 0.49.1 | 0.49.0 | 0.48.0 | 0.47.2 | 0.47.1 | 0.47.0 | 0.46.3 | 0.46.2 | 0.46.1 | 0.46.0 | 0.45.0 | 0.44.0 | 0.43.7 | 0.43.6 | 0.43.5 | 0.43.4 | 0.43.3 | 0.43.2 | 0.43.1 | 0.43.0 | 0.42.1 | 0.42.0 | 0.41.1 | 0.41.0 | 0.40.2 | 0.40.1 | 0.40.0 | 0.39.0 | 0.38.1 | 0.38.0 | 0.37.0 | 0.36.0 | 0.35.0 | 0.34.1 | 0.34.0 | 0.33.0 | 0.32.0 | 0.31.0 | 0.30.0 | -+========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+ -| 0.52.1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.52.0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.51.0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.50.1 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.50.0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.49.2 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.49.1 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.49.0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.48.0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.47.2 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.47.1 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.47.0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.46.3 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.46.2 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.46.1 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.46.0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.45.0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.44.0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.43.7 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.43.6 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.43.5 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.43.4 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.43.3 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.43.2 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.43.1 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.43.0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.42.1 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.42.0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.41.1 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.41.0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.40.2 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.40.1 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.40.0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.39.0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.38.1 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.38.0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.37.0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.36.0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.35.0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.34.1 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.34.0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.33.0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.32.0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.31.0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.30.0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| | 0.53.0 | 0.52.1 | 0.52.0 | 0.51.0 | 0.50.1 | 0.50.0 | 0.49.2 | 0.49.1 | 0.49.0 | 0.48.0 | 0.47.2 | 0.47.1 | 0.47.0 | 0.46.3 | 0.46.2 | 0.46.1 | 0.46.0 | 0.45.0 | 0.44.0 | 0.43.7 | 0.43.6 | 0.43.5 | 0.43.4 | 0.43.3 | 0.43.2 | 0.43.1 | 0.43.0 | 0.42.1 | 0.42.0 | 0.41.1 | 0.41.0 | 0.40.2 | 0.40.1 | 0.40.0 | ++========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+ +| 0.53.0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.52.1 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.52.0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.51.0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.50.1 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.50.0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.49.2 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.49.1 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.49.0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.48.0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.47.2 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.47.1 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.47.0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.46.3 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.46.2 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.46.1 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.46.0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.45.0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.44.0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.43.7 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.43.6 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.43.5 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.43.4 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.43.3 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.43.2 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.43.1 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.43.0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.42.1 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.42.0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.41.1 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.41.0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.40.2 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.40.1 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.40.0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+