diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 6ee800108..d830f7187 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -14,15 +14,10 @@ on: jobs: check-semantic-version: if: "!contains(github.event.head_commit.message, 'skipci')" - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - uses: octue/check-semantic-version@1.0.0.beta-9 - with: - path: pyproject.toml - breaking_change_indicated_by: minor + uses: octue/workflows/.github/workflows/check-semantic-version.yml@main + with: + path: pyproject.toml + breaking_change_indicated_by: minor run-tests: if: "!contains(github.event.head_commit.message, 'skipci')" @@ -68,10 +63,11 @@ jobs: run: tox -vv -e py - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: files: coverage.xml fail_ci_if_error: true + token: ${{ secrets.CODECOV_TOKEN }} test-publish: if: "!contains(github.event.head_commit.message, 'skipci')" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 38b4f8d35..eb6b95e38 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -59,10 +59,11 @@ jobs: run: tox -vv -e py - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: files: coverage.xml fail_ci_if_error: false + token: ${{ secrets.CODECOV_TOKEN }} outputs: package_version: ${{ steps.get-package-version.outputs.PACKAGE_VERSION }} diff --git a/.github/workflows/update-pull-request.yml b/.github/workflows/update-pull-request.yml index 30f1c7150..d77ca388b 100644 --- a/.github/workflows/update-pull-request.yml +++ b/.github/workflows/update-pull-request.yml @@ -6,21 +6,13 @@ name: update-pull-request -on: pull_request +on: [pull_request] jobs: description: - if: "!contains(github.event.pull_request.body, '')" - runs-on: ubuntu-latest - steps: - - uses: octue/generate-pull-request-description@1.0.0.beta-2 - id: pr-description - with: - pull_request_url: ${{ github.event.pull_request.url }} - api_token: ${{ secrets.GITHUB_TOKEN }} - - - name: Update pull request body - uses: riskledger/update-pr-description@v2 - with: - body: ${{ steps.pr-description.outputs.pull_request_description }} - token: ${{ secrets.GITHUB_TOKEN }} + uses: octue/workflows/.github/workflows/generate-pull-request-description.yml@main + secrets: + token: ${{ secrets.GITHUB_TOKEN }} + permissions: + contents: read + pull-requests: write diff --git a/LICENSE b/LICENSE index 966e776cf..2ebc4c551 100644 --- a/LICENSE +++ b/LICENSE @@ -3,7 +3,7 @@ octue-sdk-python Application SDK for python-based apps on the Octue platform MIT License -Copyright (c) 2017-2022 Octue Ltd +Copyright (c) 2017-2024 Octue Ltd Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/docs/source/asking_questions.rst b/docs/source/asking_questions.rst index 7ca817221..acc4b6448 100644 --- a/docs/source/asking_questions.rst +++ b/docs/source/asking_questions.rst @@ -4,11 +4,27 @@ Asking services questions ========================= -How to ask a question -===================== +What is a question? +=================== +A question is a set of data (input values and/or an input manifest) sent to a child for processing/analysis. Questions +can be: + +- **Synchronous ("ask-and-wait"):** A question whose answer is waited for in real time + +- **Asynchronous ("fire-and-forget"):** A question whose answer is not waited for and is instead retrieved later. There + are two types: + + - **Regular:** Responses to these questions are automatically stored in an event store where they can be :ref:`retrieved using the Octue SDK ` + + - **Push endpoint:** Responses to these questions are pushed to an HTTP endpoint for asynchronous handling using Octue's + `django-twined `_ or custom logic in your own webserver. + Questions are always asked to a *revision* of a service. You can ask a service a question if you have its -:ref:`SRUID `, project name, and the necessary permissions. The question is formed of input values -and/or an input manifest. +:ref:`SRUID `, project name, and the necessary permissions. + + +Asking a question +================= .. code-block:: python @@ -47,19 +63,133 @@ You can also set the following options when you call :mod:`Child.ask `) - ``timeout`` - how long in seconds to wait for an answer (``None`` by default - i.e. don't time out) +Exceptions raised by a child +---------------------------- If a child raises an exception while processing your question, the exception will always be forwarded and re-raised in your local service or python session. You can handle exceptions in whatever way you like. -If setting a timeout, bear in mind that the question has to reach the child, the child has to run its analysis on -the inputs sent to it (this most likely corresponds to the dominant part of the wait time), and the answer has to be -sent back to the parent. If you're not sure how long a particular analysis might take, it's best to set the timeout to -``None`` initially or ask the owner/maintainer of the child for an estimate. +Timeouts +-------- +If setting a timeout, bear in mind that the question has to reach the child, the child has to run its analysis on the +inputs sent to it (this will most likely make up the dominant part of the wait time), and the answer has to be sent back +to the parent. If you're not sure how long a particular analysis might take, it's best to set the timeout to ``None`` +initially or ask the owner/maintainer of the child for an estimate. + + +.. _retrieving_asynchronous_answers: + +Retrieving answers to asynchronous questions +============================================ +To retrieve results and other events from the processing of a question later, make sure you have the permissions to +access the event store and run: + +.. code-block:: python + + from octue.cloud.pub_sub.bigquery import get_events + + events = get_events( + table_id="your-project.your-dataset.your-table", + sender="octue/test-service:1.0.0", + question_uuid="53353901-0b47-44e7-9da3-a3ed59990a71", + ) + + +**Options** + +- ``kind`` - Only retrieve this kind of event if present (e.g. "result") +- ``include_attributes`` - If ``True``, retrieve all the events' attributes as well +- ``include_backend_metadata`` - If ``True``, retrieve information about the service backend that produced the event +- ``limit`` - If set to a positive integer, limit the number of events returned to this + + +.. collapse:: See an example output here... + + .. code-block:: python + + >>> events + [ + { + "event": { + "datetime": "2024-03-06T15:44:18.156044", + "kind": "delivery_acknowledgement" + }, + }, + { + "event": { + "kind": "log_record", + "log_record": { + "args": null, + "created": 1709739861.5949728, + "exc_info": null, + "exc_text": null, + "filename": "app.py", + "funcName": "run", + "levelname": "INFO", + "levelno": 20, + "lineno": 28, + "module": "app", + "msecs": 594.9728488922119, + "msg": "Finished example analysis.", + "name": "app", + "pathname": "/workspace/example_service_cloud_run/app.py", + "process": 2, + "processName": "MainProcess", + "relativeCreated": 8560.13798713684, + "stack_info": null, + "thread": 68328473233152, + "threadName": "ThreadPoolExecutor-0_2" + } + }, + }, + { + "event": { + "datetime": "2024-03-06T15:46:18.167424", + "kind": "heartbeat" + }, + "attributes": { + "datetime": "2024-04-11T10:46:48.236064", + "uuid": "a9de11b1-e88f-43fa-b3a4-40a590c3443f", + "order": "7", + "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", + "originator": "octue/test-service:1.0.0", + "sender": "octue/test-service:1.0.0", + "sender_type": "CHILD", + "sender_sdk_version": "0.51.0", + "recipient": "octue/another-service:3.2.1" + } + } + { + "event": { + "kind": "result", + "output_manifest": { + "datasets": { + "example_dataset": { + "files": [ + "gs://octue-sdk-python-test-bucket/example_output_datasets/example_dataset/output.dat" + ], + "id": "419bff6b-08c3-4c16-9eb1-5d1709168003", + "labels": [], + "name": "divergent-strange-gharial-of-pizza", + "path": "https://storage.googleapis.com/octue-sdk-python-test-bucket/example_output_datasets/example_dataset/.signed_metadata_files/divergent-strange-gharial-of-pizza", + "tags": {} + } + }, + "id": "a13713ae-f207-41c6-9e29-0a848ced6039", + "name": null + }, + "output_values": [1, 2, 3, 4, 5] + }, + }, + ] +---- Asking multiple questions in parallel ===================================== @@ -81,7 +211,7 @@ raised and no answers are returned. This method uses multithreading, allowing all the questions to be asked at once instead of one after another. -Options: +**Options** - If ``raise_errors=False`` is provided, answers are returned for all successful questions while unraised errors are returned for unsuccessful ones diff --git a/docs/source/conf.py b/docs/source/conf.py index 6003967a8..023694a62 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -51,7 +51,7 @@ # General information about the project. project = "Octue SDK (Python)" author = "Octue Ltd" -copyright = "2022, Octue Ltd" +copyright = "2024, Octue Ltd" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/docs/source/inter_service_compatibility.rst b/docs/source/inter_service_compatibility.rst index 86016b351..492dedff9 100644 --- a/docs/source/inter_service_compatibility.rst +++ b/docs/source/inter_service_compatibility.rst @@ -5,107 +5,87 @@ Inter-service compatibility Octue services acting as parents and children communicate with each other according to the `services communication schema `_. Up until version ``0.51.0``, services running nearly all versions of ``octue`` could communicate with each other compatibly. To allow a significant infrastructure upgrade, -version ``0.51.0`` introduced a number of breaking changes to the standard meaning services running ``0.51.0`` or -greater are only able to communicate with other services running ``0.51.0`` or greater. The table below shows which -``octue`` versions parents can run (rows) to send questions compatible with versions children are running (columns). -Note that this table does not display whether children's responses are compatible with the parent, just that a child is -able to accept a question. +version ``0.51.0`` introduced a number of breaking changes to the standard meaning services running versions ``0.51.0`` +to ``0.52.1`` are only able to communicate with other services running versions in the same range. The same applies to +services running versions ``>=0.53.0``. + +The table below shows which ``octue`` versions parents can run (rows) to send questions compatible with versions +children are running (columns). Note that this table does not display whether children's responses are compatible with +the parent, just that a child is able to accept a question. **Key** - ``0`` = incompatible - ``1`` = compatible -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| | 0.52.1 | 0.52.0 | 0.51.0 | 0.50.1 | 0.50.0 | 0.49.2 | 0.49.1 | 0.49.0 | 0.48.0 | 0.47.2 | 0.47.1 | 0.47.0 | 0.46.3 | 0.46.2 | 0.46.1 | 0.46.0 | 0.45.0 | 0.44.0 | 0.43.7 | 0.43.6 | 0.43.5 | 0.43.4 | 0.43.3 | 0.43.2 | 0.43.1 | 0.43.0 | 0.42.1 | 0.42.0 | 0.41.1 | 0.41.0 | 0.40.2 | 0.40.1 | 0.40.0 | 0.39.0 | 0.38.1 | 0.38.0 | 0.37.0 | 0.36.0 | 0.35.0 | 0.34.1 | 0.34.0 | 0.33.0 | 0.32.0 | 0.31.0 | 0.30.0 | -+========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+ -| 0.52.1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.52.0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.51.0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.50.1 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.50.0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.49.2 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.49.1 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.49.0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.48.0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.47.2 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.47.1 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.47.0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.46.3 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.46.2 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.46.1 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.46.0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.45.0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.44.0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.43.7 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.43.6 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.43.5 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.43.4 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.43.3 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.43.2 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.43.1 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.43.0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.42.1 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.42.0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.41.1 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.41.0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.40.2 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.40.1 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.40.0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.39.0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.38.1 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.38.0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.37.0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.36.0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.35.0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.34.1 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.34.0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.33.0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.32.0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.31.0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.30.0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| | 0.53.0 | 0.52.1 | 0.52.0 | 0.51.0 | 0.50.1 | 0.50.0 | 0.49.2 | 0.49.1 | 0.49.0 | 0.48.0 | 0.47.2 | 0.47.1 | 0.47.0 | 0.46.3 | 0.46.2 | 0.46.1 | 0.46.0 | 0.45.0 | 0.44.0 | 0.43.7 | 0.43.6 | 0.43.5 | 0.43.4 | 0.43.3 | 0.43.2 | 0.43.1 | 0.43.0 | 0.42.1 | 0.42.0 | 0.41.1 | 0.41.0 | 0.40.2 | 0.40.1 | 0.40.0 | ++========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+ +| 0.53.0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.52.1 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.52.0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.51.0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.50.1 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.50.0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.49.2 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.49.1 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.49.0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.48.0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.47.2 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.47.1 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.47.0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.46.3 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.46.2 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.46.1 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.46.0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.45.0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.44.0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.43.7 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.43.6 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.43.5 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.43.4 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.43.3 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.43.2 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.43.1 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.43.0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.42.1 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.42.0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.41.1 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.41.0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.40.2 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.40.1 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ +| 0.40.0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | ++--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ diff --git a/docs/source/manifest.rst b/docs/source/manifest.rst index 69eb1b235..8d00af312 100644 --- a/docs/source/manifest.rst +++ b/docs/source/manifest.rst @@ -56,11 +56,12 @@ See :doc:`here ` for more information. Receive datasets from a service ------------------------------- -Get output datasets from an Octue service from the cloud when you're ready. +Access output datasets from an Octue service from the cloud when you're ready. .. code-block:: python - answer["output_manifest"]["an_output_dataset"].files + manifest = answer["output_manifest"] + manifest["an_output_dataset"].files >>> , })> .. hint:: @@ -69,6 +70,18 @@ Get output datasets from an Octue service from the cloud when you're ready. them - the output manifest is this reference. You’ll need to use it straight away or save it to make use of it. +Download all datasets from a manifest +------------------------------------- +Download all or a subset of datasets from a manifest. + +.. code-block:: python + + manifest.download() + >>> { + "my_dataset": "/path/to/dataset" + } + + Further information =================== diff --git a/docs/source/testing_services.rst b/docs/source/testing_services.rst index 7549a4c68..df2419124 100644 --- a/docs/source/testing_services.rst +++ b/docs/source/testing_services.rst @@ -22,18 +22,18 @@ your tests: The Child Emulator ------------------ -We've written a child emulator that takes a list of messages and returns them to the parent for handling in the order -given - without contacting the real child or using Pub/Sub. Any messages a real child can produce are supported. +We've written a child emulator that takes a list of events and returns them to the parent for handling in the order +given - without contacting the real child or using Pub/Sub. Any events a real child can produce are supported. :mod:`Child ` instances can be mocked like-for-like by :mod:`ChildEmulator ` instances without the parent knowing. You can provide -the emulated messages in python or via a JSON file. +the emulated events in python or via a JSON file. Message types ------------- You can emulate any message type that your app (the parent) can handle. The table below shows what these are. +-----------------------+--------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+ -| Message type | Number of messages supported | Example | +| Message type | Number of events supported | Example | +=======================+==================================================================================================+===========================================================================================================================+ | ``log_record`` | Any number | {"type": "log_record": "log_record": {"msg": "Starting analysis."}} | +-----------------------+--------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+ @@ -50,7 +50,7 @@ You can emulate any message type that your app (the parent) can handle. The tabl - The ``log_record`` key of a ``log_record`` message is any dictionary that the ``logging.makeLogRecord`` function can convert into a log record. - The ``data`` key of a ``monitor_message`` message must be a JSON-serialised string -- Any messages after a ``result`` or ``exception`` message won't be passed to the parent because execution of the child +- Any events after a ``result`` or ``exception`` message won't be passed to the parent because execution of the child emulator will have ended. @@ -59,7 +59,7 @@ Instantiating a child emulator in python .. code-block:: python - messages = [ + events = [ { "type": "log_record", "log_record": {"msg": "Starting analysis."}, @@ -81,7 +81,7 @@ Instantiating a child emulator in python child_emulator = ChildEmulator( backend={"name": "GCPPubSubBackend", "project_name": "my-project"}, - messages=messages + events=events ) def handle_monitor_message(message): @@ -96,14 +96,14 @@ Instantiating a child emulator in python Instantiating a child emulator from a JSON file ----------------------------------------------- -You can provide a JSON file with either just messages in or with messages and some or all of the +You can provide a JSON file with either just events in or with events and some or all of the :mod:`ChildEmulator ` constructor parameters. Here's an example JSON file -with just the messages: +with just the events: .. code-block:: json { - "messages": [ + "events": [ { "type": "log_record", "log_record": {"msg": "Starting analysis."} @@ -179,7 +179,7 @@ To emulate your children in tests, patch the :mod:`Child >> [ { 'type': 'delivery_acknowledgement', @@ -257,7 +257,7 @@ You can then feed these into a child emulator to emulate one possible response o from octue.cloud.emulators import ChildEmulator - child_emulator = ChildEmulator(messages=child.received_messages) + child_emulator = ChildEmulator(events=child.received_events) child_emulator.ask(input_values=[1, 2, 3, 4]) >>> {"some": "results"} diff --git a/octue/__init__.py b/octue/__init__.py index 138e91f2c..eea4e34fb 100644 --- a/octue/__init__.py +++ b/octue/__init__.py @@ -6,6 +6,7 @@ __all__ = ("Runner",) + REPOSITORY_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) diff --git a/octue/cli.py b/octue/cli.py index 643209551..08889808f 100644 --- a/octue/cli.py +++ b/octue/cli.py @@ -10,10 +10,9 @@ import click from google import auth -from octue.cloud import storage -from octue.cloud.pub_sub import Subscription, Topic -from octue.cloud.pub_sub.service import PARENT_SENDER_TYPE, Service -from octue.cloud.service_id import convert_service_id_to_pub_sub_form, create_sruid, get_sruid_parts +from octue.cloud import pub_sub, storage +from octue.cloud.pub_sub.service import Service +from octue.cloud.service_id import create_sruid, get_sruid_parts from octue.cloud.storage import GoogleCloudStorageClient from octue.configuration import load_service_and_app_configuration from octue.definitions import MANIFEST_FILENAME, VALUES_FILENAME @@ -366,8 +365,8 @@ def create_push_subscription( expiration_time, revision_tag, ): - """Create a push subscription on Google Pub/Sub from the Octue service to the push endpoint. If a corresponding - topic doesn't exist, it will be created. The subscription name is printed on completion. + """Create a Google Pub/Sub push subscription for an Octue service for it to receive questions from parents. The + subscription name is printed on completion. PROJECT_NAME is the name of the Google Cloud project in which the subscription will be created @@ -378,29 +377,17 @@ def create_push_subscription( PUSH_ENDPOINT is the HTTP/HTTPS endpoint of the service to push to. It should be fully formed and include the 'https://' prefix """ - service_sruid = create_sruid(namespace=service_namespace, name=service_name, revision_tag=revision_tag) - pub_sub_sruid = convert_service_id_to_pub_sub_form(service_sruid) + sruid = create_sruid(namespace=service_namespace, name=service_name, revision_tag=revision_tag) - topic = Topic(name=pub_sub_sruid, project_name=project_name) - topic.create(allow_existing=True) - - if expiration_time: - expiration_time = float(expiration_time) - else: - # Convert empty string to `None`. - expiration_time = None - - subscription = Subscription( - name=pub_sub_sruid, - topic=topic, - project_name=project_name, - filter=f'attributes.sender_type = "{PARENT_SENDER_TYPE}"', + pub_sub.create_push_subscription( + project_name, + sruid, + push_endpoint, expiration_time=expiration_time, - push_endpoint=push_endpoint, + subscription_filter=f'attributes.recipient = "{sruid}" AND attributes.sender_type = "PARENT"', ) - subscription.create() - click.echo(service_sruid) + click.echo(sruid) def _add_monitor_message_to_file(path, monitor_message): diff --git a/octue/cloud/deployment/google/answer_pub_sub_question.py b/octue/cloud/deployment/google/answer_pub_sub_question.py index 167832582..703a67f5c 100644 --- a/octue/cloud/deployment/google/answer_pub_sub_question.py +++ b/octue/cloud/deployment/google/answer_pub_sub_question.py @@ -1,8 +1,7 @@ import logging -from octue.cloud.pub_sub import Topic from octue.cloud.pub_sub.service import Service -from octue.cloud.service_id import convert_service_id_to_pub_sub_form, create_sruid, get_sruid_parts +from octue.cloud.service_id import create_sruid, get_sruid_parts from octue.configuration import load_service_and_app_configuration from octue.resources.service_backends import GCPPubSubBackend from octue.runner import Runner @@ -47,9 +46,5 @@ def answer_question(question, project_name): logger.info("Analysis successfully run and response sent for question %r.", question_uuid) except BaseException as error: # noqa - service.send_exception( - topic=Topic(name=convert_service_id_to_pub_sub_form(service_sruid), project_name=project_name), - question_uuid=question_uuid, - ) - + service.send_exception(question_uuid=question_uuid, originator="UNKNOWN") logger.exception(error) diff --git a/octue/cloud/emulators/_pub_sub.py b/octue/cloud/emulators/_pub_sub.py index be9331164..ee6733ca5 100644 --- a/octue/cloud/emulators/_pub_sub.py +++ b/octue/cloud/emulators/_pub_sub.py @@ -1,20 +1,22 @@ import importlib.metadata import json import logging +from collections import defaultdict import google.api_core from octue.cloud.pub_sub import Subscription, Topic -from octue.cloud.pub_sub.service import ANSWERS_NAMESPACE, PARENT_SENDER_TYPE, Service -from octue.cloud.service_id import convert_service_id_to_pub_sub_form +from octue.cloud.pub_sub.service import PARENT_SENDER_TYPE, Service from octue.resources import Manifest +from octue.utils.dictionaries import make_minimal_dictionary from octue.utils.encoders import OctueJSONEncoder logger = logging.getLogger(__name__) -TOPICS = {} -SUBSCRIPTIONS = {} +TOPICS = set() +SUBSCRIPTIONS = set() +MESSAGES = defaultdict(list) class MockTopic(Topic): @@ -32,7 +34,7 @@ def create(self, allow_existing=False): raise google.api_core.exceptions.AlreadyExists(f"Topic {self.path!r} already exists.") if not self.exists(): - TOPICS[self.name] = [] + TOPICS.add(self.name) self._created = True def delete(self): @@ -41,7 +43,7 @@ def delete(self): :return None: """ try: - del TOPICS[self.name] + TOPICS.remove(self.name) except KeyError: pass @@ -72,7 +74,7 @@ def create(self, allow_existing=False): raise google.api_core.exceptions.AlreadyExists(f"Subscription {self.path!r} already exists.") if not self.exists(): - SUBSCRIPTIONS[self.name] = [] + SUBSCRIPTIONS.add(self.name) self._created = True def delete(self): @@ -145,8 +147,7 @@ def publish(self, topic, data, retry=None, **attributes): :param google.api_core.retry.Retry|None retry: :return MockFuture: """ - subscription_name = ".".join((get_pub_sub_resource_name(topic), ANSWERS_NAMESPACE, attributes["question_uuid"])) - SUBSCRIPTIONS[subscription_name].append(MockMessage(data=data, attributes=attributes)) + MESSAGES[attributes["question_uuid"]].append(MockMessage(data=data, attributes=attributes)) return MockFuture() @@ -190,12 +191,12 @@ def pull(self, request, timeout=None, retry=None): if self.closed: raise ValueError("ValueError: Cannot invoke RPC: Channel closed!") - subscription_name = get_pub_sub_resource_name(request["subscription"]) + question_uuid = request["subscription"].split(".")[-1] try: return MockPullResponse( received_messages=[ - MockMessageWrapper(message=SUBSCRIPTIONS[subscription_name].pop(0)), + MockMessageWrapper(message=MESSAGES[question_uuid].pop(0)), ] ) @@ -241,6 +242,13 @@ def __init__(self, message): self.message = message self.ack_id = None + def __repr__(self): + """Represent the mock message wrapper as a string. + + :return str: + """ + return f"<{type(self).__name__}(message={self.message})>" + class MockMessage: """A mock Pub/Sub message containing serialised data and any number of attributes. @@ -319,6 +327,7 @@ def ask( save_diagnostics="SAVE_DIAGNOSTICS_ON_CRASH", question_uuid=None, push_endpoint=None, + asynchronous=False, timeout=86400, parent_sdk_version=importlib.metadata.version("octue"), ): @@ -334,6 +343,7 @@ def ask( :param str save_diagnostics: :param str|None question_uuid: :param str|None push_endpoint: + :param bool asynchronous: :param float|None timeout: :return MockFuture, str: """ @@ -347,38 +357,37 @@ def ask( save_diagnostics=save_diagnostics, question_uuid=question_uuid, push_endpoint=push_endpoint, + asynchronous=asynchronous, timeout=timeout, ) # Delete question from messages sent to subscription so the parent doesn't pick it up as a response message. We # do this as subscription filtering isn't implemented in this set of mocks. - subscription_name = ".".join((convert_service_id_to_pub_sub_form(service_id), ANSWERS_NAMESPACE, question_uuid)) - SUBSCRIPTIONS["octue.services." + subscription_name].pop(0) + MESSAGES[question_uuid].pop(0) - question = {"kind": "question"} - - if input_values is not None: - question["input_values"] = input_values + question = make_minimal_dictionary(kind="question", input_values=input_values, children=children) # Ignore any errors from the answering service as they will be raised on the remote service in practice, not # locally as is done in this mock. if input_manifest is not None: question["input_manifest"] = input_manifest.to_primitive() - if children is not None: - question["children"] = children - try: self.children[service_id].answer( MockMessage.from_primitive( data=question, attributes={ - "sender_type": PARENT_SENDER_TYPE, + "datetime": "2024-04-11T10:46:48.236064", + "uuid": "a9de11b1-e88f-43fa-b3a4-40a590c3443f", "question_uuid": question_uuid, "forward_logs": subscribe_to_logs, - "version": parent_sdk_version, "save_diagnostics": save_diagnostics, - "message_number": 0, + "order": 0, + "originator": self.id, + "sender": self.id, + "sender_type": PARENT_SENDER_TYPE, + "sender_sdk_version": parent_sdk_version, + "recipient": service_id, }, ) ) diff --git a/octue/cloud/emulators/child.py b/octue/cloud/emulators/child.py index 71d59a61b..7614c5fd8 100644 --- a/octue/cloud/emulators/child.py +++ b/octue/cloud/emulators/child.py @@ -1,6 +1,7 @@ import copy import json import logging +import warnings from unittest.mock import patch from octue.cloud import EXCEPTIONS_MAPPING @@ -13,19 +14,19 @@ class ChildEmulator: - """An emulator for the `octue.resources.child.Child` class that sends the given messages to the parent for handling - without contacting the real child or using Pub/Sub. Any messages a real child could produce are supported. `Child` + """An emulator for the `octue.resources.child.Child` class that sends the given events to the parent for handling + without contacting the real child or using Pub/Sub. Any events a real child could produce are supported. `Child` instances can be replaced/mocked like-for-like by `ChildEmulator` without the parent knowing. :param str|None id: the ID of the child; a UUID is generated if none is provided :param dict|None backend: a dictionary including the key "name" with a value of the name of the type of backend (e.g. "GCPPubSubBackend") and key-value pairs for any other parameters the chosen backend expects; a mock backend is used if none is provided :param str internal_service_name: the name to give to the internal service used to ask questions to the child - :param list(dict)|None messages: the list of messages to send to the parent + :param list(dict)|None events: the list of events to send to the parent :return None: """ - def __init__(self, id=None, backend=None, internal_service_name="local/local:local", messages=None): - self.messages = messages or [] + def __init__(self, id=None, backend=None, internal_service_name="local/local:local", events=None): + self.events = events or [] backend = copy.deepcopy(backend or {"name": "GCPPubSubBackend", "project_name": "emulated-project"}) backend_type_name = backend.pop("name") @@ -40,7 +41,7 @@ def __init__(self, id=None, backend=None, internal_service_name="local/local:loc children={self._child.id: self._child}, ) - self._message_handlers = { + self._event_handlers = { "delivery_acknowledgement": self._handle_delivery_acknowledgement, "heartbeat": self._handle_heartbeat, "log_record": self._handle_log_record, @@ -49,7 +50,7 @@ def __init__(self, id=None, backend=None, internal_service_name="local/local:loc "result": self._handle_result, } - self._valid_message_kinds = set(self._message_handlers.keys()) + self._valid_event_kinds = set(self._event_handlers.keys()) @classmethod def from_file(cls, path): @@ -62,11 +63,23 @@ def from_file(cls, path): with open(path) as f: serialised_child_emulator = json.load(f) + if "messages" in serialised_child_emulator: + events = serialised_child_emulator["messages"] + + warnings.warn( + "Use of 'messages' as a key in an events JSON file for a child emulator is deprecated, and support for " + "it will be removed soon. Please use 'events' for the key instead.", + category=DeprecationWarning, + ) + + else: + events = serialised_child_emulator.get("events") + return cls( id=serialised_child_emulator.get("id"), backend=serialised_child_emulator.get("backend"), internal_service_name=serialised_child_emulator.get("internal_service_name"), - messages=serialised_child_emulator.get("messages"), + events=events, ) def __repr__(self): @@ -77,12 +90,12 @@ def __repr__(self): return f"<{type(self).__name__}({self.id!r})>" @property - def received_messages(self): - """Get the messages received from the child. + def received_events(self): + """Get the events received from the child. :return list(dict): """ - return self._parent.received_messages + return self._parent.received_events def ask( self, @@ -91,21 +104,25 @@ def ask( subscribe_to_logs=True, allow_local_files=False, handle_monitor_message=None, - record_messages=True, + record_events=True, question_uuid=None, + push_endpoint=None, + asynchronous=False, timeout=86400, ): - """Ask the child emulator a question and receive its emulated response messages. Unlike a real child, the input + """Ask the child emulator a question and receive its emulated response events. Unlike a real child, the input values and manifest are not validated against the schema in the child's twine as it is only available to the - real child. Hence, the input values and manifest do not affect the messages returned by the emulator. + real child. Hence, the input values and manifest do not affect the events returned by the emulator. :param any|None input_values: any input values for the question :param octue.resources.manifest.Manifest|None input_manifest: an input manifest of any datasets needed for the question :param bool subscribe_to_logs: if `True`, subscribe to logs from the child and handle them with the local log handlers :param bool allow_local_files: if `True`, allow the input manifest to contain references to local files - this should only be set to `True` if the child will have access to these local files :param callable|None handle_monitor_message: a function to handle monitor messages (e.g. send them to an endpoint for plotting or displaying) - this function should take a single JSON-compatible python primitive as an argument (note that this could be an array or object) - :param bool record_messages: if `True`, record messages received from the child in the `received_messages` property + :param bool record_events: if `True`, record events received from the child in the `received_events` property :param str|None question_uuid: the UUID to use for the question if a specific one is needed; a UUID is generated if not + :param str|None push_endpoint: if answers to the question should be pushed to an endpoint, provide its URL here (the returned subscription will be a push subscription); if not, leave this as `None` + :param bool asynchronous: if `True`, don't create an answer subscription :param float timeout: time in seconds to wait for an answer before raising a timeout error :raise TimeoutError: if the timeout is exceeded while waiting for an answer :return dict: a dictionary containing the keys "output_values" and "output_manifest" @@ -120,12 +137,14 @@ def ask( subscribe_to_logs=subscribe_to_logs, allow_local_files=allow_local_files, question_uuid=question_uuid, + push_endpoint=push_endpoint, + asynchronous=asynchronous, ) return self._parent.wait_for_answer( subscription, handle_monitor_message=handle_monitor_message, - record_messages=record_messages, + record_events=record_events, timeout=timeout, ) @@ -139,7 +158,7 @@ def _emulate_analysis( handle_monitor_message, save_diagnostics, ): - """Emulate analysis of a question by handling the messages given at instantiation in the order given. + """Emulate analysis of a question by handling the events given at instantiation in the order given. :param str|None analysis_id: UUID of analysis :param str|dict|None input_values: any input values for the question @@ -150,12 +169,12 @@ def _emulate_analysis( :param str save_diagnostics: must be one of {"SAVE_DIAGNOSTICS_OFF", "SAVE_DIAGNOSTICS_ON_CRASH", "SAVE_DIAGNOSTICS_ON"}; if turned on, allow the input values and manifest (and its datasets) to be saved by the child either all the time or just if the analysis fails :return octue.resources.analysis.Analysis: """ - for message in self.messages: - self._validate_message(message) - handler = self._message_handlers[message["kind"]] + for event in self.events: + self._validate_event(event) + handler = self._event_handlers[event["kind"]] result = handler( - message, + event, analysis_id=analysis_id, input_values=input_values, input_manifest=input_manifest, @@ -168,7 +187,7 @@ def _emulate_analysis( if result: return result - # If no result message is included in the given messages, return an empty analysis. + # If no result event is included in the given events, return an empty analysis. return Analysis( id=analysis_id, twine={}, @@ -179,59 +198,56 @@ def _emulate_analysis( output_manifest=None, ) - def _validate_message(self, message): - """Validate the given message to ensure it can be handled. + def _validate_event(self, event): + """Validate the given event to ensure it can be handled. - :param dict message: - :raise TypeError: if the message isn't a dictionary - :raise ValueError: if the message doesn't contain a 'kind' key or if the 'kind' key maps to an invalid value + :param dict event: + :raise TypeError: if the event isn't a dictionary + :raise ValueError: if the event doesn't contain a 'kind' key or if the 'kind' key maps to an invalid value :return None: """ - if not isinstance(message, dict): - raise TypeError("Each message must be a dictionary.") + if not isinstance(event, dict): + raise TypeError("Each event must be a dictionary.") - if "kind" not in message: - raise ValueError( - f"Each message must contain a 'kind' key mapping to one of: {self._valid_message_kinds!r}." - ) + if "kind" not in event: + raise ValueError(f"Each event must contain a 'kind' key mapping to one of: {self._valid_event_kinds!r}.") - if message["kind"] not in self._valid_message_kinds: + if event["kind"] not in self._valid_event_kinds: raise ValueError( - f"{message['kind']!r} is an invalid message kind for the ChildEmulator. The valid kinds are: " - f"{self._valid_message_kinds!r}." + f"{event['kind']!r} is an invalid event kind for the ChildEmulator. The valid kinds are: " + f"{self._valid_event_kinds!r}." ) - def _handle_delivery_acknowledgement(self, message, **kwargs): - """A no-operation handler for delivery acknowledgement messages (these messages are ignored by the child - emulator). + def _handle_delivery_acknowledgement(self, event, **kwargs): + """A no-operation handler for delivery acknowledgement events (these events are ignored by the child emulator). - :param dict message: a dictionary containing the key "datetime" + :param dict event: a dictionary containing the key "datetime" :param kwargs: this should be empty :return None: """ - logger.warning("Delivery acknowledgement messages are ignored by the ChildEmulator.") + logger.warning("Delivery acknowledgement events are ignored by the ChildEmulator.") - def _handle_heartbeat(self, message, **kwargs): - """A no-operation handler for heartbeat messages (these messages are ignored by the child emulator). + def _handle_heartbeat(self, event, **kwargs): + """A no-operation handler for heartbeat events (these events are ignored by the child emulator). - :param dict message: a dictionary containing the key "datetime" + :param dict event: a dictionary containing the key "datetime" :param kwargs: this should be empty :return None: """ - logger.warning("Heartbeat messages are ignored by the ChildEmulator.") + logger.warning("Heartbeat events are ignored by the ChildEmulator.") - def _handle_log_record(self, message, **kwargs): - """Convert the given message into a log record and pass it to the log handler. + def _handle_log_record(self, event, **kwargs): + """Convert the given event into a log record and pass it to the log handler. - :param dict message: a dictionary containing a "log_record" key whose value is a dictionary representing a log record + :param dict event: a dictionary containing a "log_record" key whose value is a dictionary representing a log record :param kwargs: this should be empty - :raise TypeError: if the message can't be converted to a log record + :raise TypeError: if the event can't be converted to a log record :return None: """ try: - log_record = message["log_record"] + log_record = event["log_record"] except KeyError: - raise ValueError("Log record messages must include a 'log_record' key.") + raise ValueError("Log record events must include a 'log_record' key.") try: log_record["levelno"] = log_record.get("levelno", 20) @@ -241,52 +257,52 @@ def _handle_log_record(self, message, **kwargs): except Exception: raise TypeError( - "The 'log_record' key in a log record message must map to a dictionary that can be converted by " + "The 'log_record' key in a log record event must map to a dictionary that can be converted by " "`logging.makeLogRecord` to a `logging.LogRecord` instance." ) - def _handle_monitor_message(self, message, **kwargs): + def _handle_monitor_message(self, event, **kwargs): """Handle a monitor message with the given handler. - :param dict message: a dictionary containing a "data" key mapped to a JSON-encoded string representing a monitor message. This monitor message will be handled by the monitor message handler + :param dict event: a dictionary containing a "data" key mapped to a JSON-encoded string representing a monitor message. This monitor message will be handled by the monitor message handler :param kwargs: must include the "handle_monitor_message" key :return None: """ - kwargs.get("handle_monitor_message")(message["data"]) + kwargs.get("handle_monitor_message")(event["data"]) - def _handle_exception(self, message, **kwargs): + def _handle_exception(self, event, **kwargs): """Raise the given exception. - :param dict message: a dictionary representing the exception to be raised; it must include the "exception_type" and "exception_message" keys + :param dict event: a dictionary representing the exception to be raised; it must include the "exception_type" and "exception_message" keys :param kwargs: this should be empty :raise ValueError: if the given exception cannot be raised :return None: """ - if "exception_type" not in message or "exception_message" not in message: + if "exception_type" not in event or "exception_message" not in event: raise ValueError( "The exception must be given as a dictionary containing the keys 'exception_type' and " "'exception_message'." ) try: - exception_type = EXCEPTIONS_MAPPING[message["exception_type"]] + exception_type = EXCEPTIONS_MAPPING[event["exception_type"]] # Allow unknown exception types to still be raised. except KeyError: - exception_type = type(message["exception_type"], (Exception,), {}) + exception_type = type(event["exception_type"], (Exception,), {}) - raise exception_type(message["exception_message"]) + raise exception_type(event["exception_message"]) - def _handle_result(self, message, **kwargs): + def _handle_result(self, event, **kwargs): """Return the result as an `Analysis` instance. - :param dict message: a dictionary containing an "output_values" key and an "output_manifest" key + :param dict event: a dictionary containing an "output_values" key and an "output_manifest" key :param kwargs: must contain the keys "analysis_id", "handle_monitor_message", "input_values", and "input_manifest" :raise ValueError: if the result doesn't contain the "output_values" and "output_manifest" keys :return octue.resources.analysis.Analysis: an `Analysis` instance containing the emulated outputs """ input_manifest = kwargs.get("input_manifest") - output_manifest = message.get("output_manifest") + output_manifest = event.get("output_manifest") if input_manifest and not isinstance(input_manifest, Manifest): input_manifest = Manifest.deserialise(input_manifest) @@ -300,7 +316,7 @@ def _handle_result(self, message, **kwargs): handle_monitor_message=kwargs["handle_monitor_message"], input_values=kwargs["input_values"], input_manifest=input_manifest, - output_values=message.get("output_values"), + output_values=event.get("output_values"), output_manifest=output_manifest, ) @@ -316,7 +332,7 @@ def __init__(self): patches=[ patch("octue.cloud.pub_sub.service.Topic", new=MockTopic), patch("octue.cloud.pub_sub.service.Subscription", new=MockSubscription), - patch("octue.cloud.pub_sub.message_handler.SubscriberClient", new=MockSubscriber), + patch("octue.cloud.pub_sub.events.SubscriberClient", new=MockSubscriber), patch("google.cloud.pubsub_v1.SubscriberClient", new=MockSubscriber), ] ) diff --git a/octue/cloud/emulators/cloud_storage.py b/octue/cloud/emulators/cloud_storage.py index 1312a85d6..5aa72a9ce 100644 --- a/octue/cloud/emulators/cloud_storage.py +++ b/octue/cloud/emulators/cloud_storage.py @@ -5,7 +5,7 @@ from contextlib import closing from gcp_storage_emulator.server import create_server -from google.cloud.storage.blob import _API_ACCESS_ENDPOINT +from google.cloud.storage._helpers import _get_default_storage_base_url # Silence the GCP storage emulator logger below `ERROR` level messages. @@ -100,12 +100,18 @@ def stopTestRun(self): del os.environ[self.STORAGE_EMULATOR_HOST_ENVIRONMENT_VARIABLE_NAME] -def mock_generate_signed_url(blob, expiration=datetime.timedelta(days=7), **kwargs): +def mock_generate_signed_url( + blob, + expiration=datetime.timedelta(days=7), + api_access_endpoint=_get_default_storage_base_url(), + **kwargs, +): """Mock generating a signed URL for a Google Cloud Storage blob. Signed URLs can't currently be generated when using workload identity federation, which we use for our CI tests. :param google.cloud.storage.blob.Blob blob: :param datetime.datetime|datetime.timedelta expiration: + :param str api_access_endpoint: :return str: """ mock_signed_query_parameter = ( @@ -113,5 +119,5 @@ def mock_generate_signed_url(blob, expiration=datetime.timedelta(days=7), **kwar f"roject.iam.gserviceaccount.com&Signature=mock-signature" ) - base_url = "/".join((kwargs.get("api_access_endpoint", _API_ACCESS_ENDPOINT), blob.bucket.name, blob.name)) + base_url = "/".join((api_access_endpoint, blob.bucket.name, blob.name)) return base_url + mock_signed_query_parameter diff --git a/octue/cloud/events/__init__.py b/octue/cloud/events/__init__.py new file mode 100644 index 000000000..66898c235 --- /dev/null +++ b/octue/cloud/events/__init__.py @@ -0,0 +1 @@ +OCTUE_SERVICES_PREFIX = "octue.services" diff --git a/octue/cloud/events/counter.py b/octue/cloud/events/counter.py new file mode 100644 index 000000000..5edc208a1 --- /dev/null +++ b/octue/cloud/events/counter.py @@ -0,0 +1,35 @@ +class EventCounter: + """A mutable counter for keeping track of the emission order of events. This is used in the `Service` class instead + of an integer because it is mutable and can be passed to the `Service._emit_event` method and incremented as + events are emitted. + + :return None: + """ + + def __init__(self): + self.count = 0 + + def __iadd__(self, other): + """Increment the counter by an integer. + + :return octue.cloud.events.counter.EventCounter: the event counter with its count updated + """ + if not isinstance(other, int): + raise ValueError(f"Event counters can only be incremented by an integer; received {other!r}.") + + self.count += other + return self + + def __int__(self): + """Get the counter as an integer. + + :return int: the counter as an integer + """ + return int(self.count) + + def __repr__(self): + """Represent the counter as a string. + + :return str: the counter represented as a string. + """ + return f"<{type(self).__name__}(count={self.count})" diff --git a/octue/cloud/events/handler.py b/octue/cloud/events/handler.py new file mode 100644 index 000000000..1eff9fc9c --- /dev/null +++ b/octue/cloud/events/handler.py @@ -0,0 +1,373 @@ +import abc +import importlib.metadata +import logging +import math +import os +import re +import time +from datetime import datetime + +from octue.cloud import EXCEPTIONS_MAPPING +from octue.cloud.events.validation import SERVICE_COMMUNICATION_SCHEMA, is_event_valid +from octue.definitions import GOOGLE_COMPUTE_PROVIDERS +from octue.log_handlers import COLOUR_PALETTE +from octue.resources.manifest import Manifest + + +logger = logging.getLogger(__name__) + + +if os.environ.get("COMPUTE_PROVIDER", "UNKNOWN") in GOOGLE_COMPUTE_PROVIDERS: + # Google Cloud logs don't support colour currently - provide a no-operation function. + colourise = lambda string, text_colour=None, background_colour=None: string +else: + from octue.utils.colour import colourise + + +PARENT_SDK_VERSION = importlib.metadata.version("octue") + + +class AbstractEventHandler: + """An abstract event handler for Octue service events that: + - Provide handlers for the Octue service event kinds (see https://strands.octue.com/octue/service-communication) + - Handles received events in the order specified by the `order` attribute + - Skips missing events after a set time and carries on handling from the next available event + + To create a concrete handler for a specific service/communication backend synchronously or asynchronously, inherit + from this class and add the `handle_events` and `_extract_event_and_attributes` methods. + + :param octue.cloud.pub_sub.service.Service recipient: the `Service` instance that's receiving the events + :param callable|None handle_monitor_message: a function to handle monitor messages (e.g. send them to an endpoint for plotting or displaying) - this function should take a single JSON-compatible python primitive + :param bool record_events: if `True`, record received events in the `received_events` attribute + :param dict|None event_handlers: a mapping of event type names to callables that handle each type of event. The handlers must not mutate the events. + :param dict schema: the JSON schema to validate events against + :param int|float skip_missing_events_after: the number of seconds after which to skip any events if they haven't arrived but subsequent events have + :param bool only_handle_result: if `True`, skip non-result events and only handle the "result" event when received + :return None: + """ + + def __init__( + self, + recipient, + handle_monitor_message=None, + record_events=True, + event_handlers=None, + schema=SERVICE_COMMUNICATION_SCHEMA, + skip_missing_events_after=10, + only_handle_result=False, + ): + self.recipient = recipient + self.handle_monitor_message = handle_monitor_message + self.record_events = record_events + self.schema = schema + self.only_handle_result = only_handle_result + + # These are set when the first event is received. + self.question_uuid = None + self.child_sruid = None + self.child_sdk_version = None + + self.waiting_events = None + self.handled_events = [] + self._previous_event_number = -1 + self._start_time = None + + self.skip_missing_events_after = skip_missing_events_after + self._missing_event_detection_time = None + self._earliest_waiting_event_number = math.inf + + self._event_handlers = event_handlers or { + "delivery_acknowledgement": self._handle_delivery_acknowledgement, + "heartbeat": self._handle_heartbeat, + "monitor_message": self._handle_monitor_message, + "log_record": self._handle_log_message, + "exception": self._handle_exception, + "result": self._handle_result, + } + + self._log_message_colours = [COLOUR_PALETTE[1], *COLOUR_PALETTE[3:]] + + @property + def awaiting_missing_event(self): + """Check if the event handler is currently waiting for a missing event. + + :return bool: `True` if the event handler is currently waiting for a missing event + """ + return self._missing_event_detection_time is not None + + @property + def time_since_missing_event(self): + """Get the amount of time elapsed since the last missing event was detected. If no missing events have been + detected or they've already been skipped, `None` is returned. + + :return float|None: + """ + if not self.awaiting_missing_event: + return None + + return time.perf_counter() - self._missing_event_detection_time + + @abc.abstractmethod + def handle_events(self, *args, **kwargs): + """Handle events and return a handled "result" event once one is received. This method must be overridden but + can have any arguments. The first thing it should do is call `super().handle_events()`. + + :return dict: the handled final result + """ + self.reset() + + def reset(self): + """Reset the handler to be ready to handle a new stream of events. + + :return None: + """ + self._start_time = time.perf_counter() + self.waiting_events = {} + self._previous_event_number = -1 + + @abc.abstractmethod + def _extract_event_and_attributes(self, container): + """Extract an event and its attributes from the event container. This method must be overridden. + + :param any container: the container of the event (e.g. a Pub/Sub message) + :return (any, dict): the event and its attributes (both must conform to the service communications event schema) + """ + pass + + def _extract_and_enqueue_event(self, container): + """Extract an event from its container, validate it, and add it to `self.waiting_events` if it's valid. + + :param any container: the container of the event (e.g. a Pub/Sub message) + :return None: + """ + try: + event, attributes = self._extract_event_and_attributes(container) + except Exception: + event = None + attributes = {} + + # Don't assume the presence of specific attributes before validation. + child_sdk_version = attributes.get("sender_sdk_version") + + if not is_event_valid( + event=event, + attributes=attributes, + recipient=self.recipient, + parent_sdk_version=PARENT_SDK_VERSION, + child_sdk_version=child_sdk_version, + schema=self.schema, + ): + return + + # Get the child's SRUID and Octue SDK version from the first event. + if not self.child_sdk_version: + self.question_uuid = attributes["question_uuid"] + self.child_sruid = attributes["sender"] + self.child_sdk_version = attributes["sender_sdk_version"] + + logger.debug("%r: Received an event related to question %r.", self.recipient, self.question_uuid) + order = attributes["order"] + + if order in self.waiting_events: + logger.warning( + "%r: Event with duplicate order %d received for question %r - overwriting original event.", + self.recipient, + order, + self.question_uuid, + ) + + self.waiting_events[order] = event + + def _attempt_to_handle_waiting_events(self): + """Attempt to handle any events waiting in `self.waiting_events`. If these events aren't consecutive to the + last handled event (i.e. if events have been received out of order and the next in-order event hasn't been + received yet), just return. After the missing event wait time has passed, if this set of missing events + haven't arrived but subsequent ones have, skip to the earliest waiting event and continue from there. + + :return any|None: either a handled non-`None` "result" event, or `None` if nothing was returned by the event handlers or if the next in-order event hasn't been received yet + """ + # Handle the case where no events (or no valid events) have been received. + if not self.waiting_events: + logger.debug("No events (or no valid events) were received.") + return + + self._earliest_waiting_event_number = min(self.waiting_events.keys()) + + while self.waiting_events: + try: + # If the next consecutive event has been received: + event = self.waiting_events.pop(self._previous_event_number + 1) + + # If the next consecutive event hasn't been received: + except KeyError: + # Start the missing event timer if it isn't already running. + if not self.awaiting_missing_event: + self._missing_event_detection_time = time.perf_counter() + + if self.time_since_missing_event > self.skip_missing_events_after: + event = self._skip_to_earliest_waiting_event() + + # Declare there are no more missing events. + self._missing_event_detection_time = None + + if not event: + return + + else: + return + + result = self._handle_event(event) + + if result is not None: + return result + + def _skip_to_earliest_waiting_event(self): + """Get the earliest waiting event and set the event handler up to continue from it. + + :return dict|None: the earliest waiting event if there is one + """ + try: + event = self.waiting_events.pop(self._earliest_waiting_event_number) + except KeyError: + return + + number_of_missing_events = self._earliest_waiting_event_number - self._previous_event_number - 1 + + # Let the event handler know it can handle the next earliest event. + self._previous_event_number = self._earliest_waiting_event_number - 1 + + logger.warning( + "%r: %d consecutive events missing for question %r after %ds - skipping to next earliest waiting event " + "(event %d).", + self.recipient, + number_of_missing_events, + self.question_uuid, + self.skip_missing_events_after, + self._earliest_waiting_event_number, + ) + + return event + + def _handle_event(self, event): + """Pass an event to its handler and update the previous event number. + + :param dict event: the event to handle + :return dict|None: the output of the event (this should be `None` unless the event is a "result" event) + """ + self._previous_event_number += 1 + + if self.record_events: + self.handled_events.append(event) + + if self.only_handle_result and event["kind"] != "result": + return + + handler = self._event_handlers[event["kind"]] + return handler(event) + + def _handle_delivery_acknowledgement(self, event): + """Log that the question was delivered. + + :param dict event: + :return None: + """ + logger.info("%r's question was delivered at %s.", self.recipient, event["datetime"]) + + def _handle_heartbeat(self, event): + """Record the time the heartbeat was received. + + :param dict event: + :return None: + """ + self._last_heartbeat = datetime.now() + logger.info( + "%r: Received a heartbeat from service %r for question %r.", + self.recipient, + self.child_sruid, + self.question_uuid, + ) + + def _handle_monitor_message(self, event): + """Send the monitor message to the handler if one has been provided. + + :param dict event: + :return None: + """ + logger.debug( + "%r: Received a monitor message from service %r for question %r.", + self.recipient, + self.child_sruid, + self.question_uuid, + ) + + if self.handle_monitor_message is not None: + self.handle_monitor_message(event["data"]) + + def _handle_log_message(self, event): + """Deserialise the event into a log record and pass it to the local log handlers. The child's SRUID and the + question UUID are added to the start of the log message, and the SRUIDs of any subchildren called by the child + are each coloured differently. + + :param dict event: + :return None: + """ + record = logging.makeLogRecord(event["log_record"]) + + # Add information about the immediate child sending the event and colour it with the first colour in the + # colour palette. + immediate_child_analysis_section = colourise( + f"[{self.child_sruid} | analysis-{self.question_uuid}]", + text_colour=self._log_message_colours[0], + ) + + # Colour any analysis sections from children of the immediate child with the rest of the colour palette. + subchild_analysis_sections = [section.strip("[") for section in re.split("] ", record.msg)] + final_message = subchild_analysis_sections.pop(-1) + + for i in range(len(subchild_analysis_sections)): + subchild_analysis_sections[i] = colourise( + "[" + subchild_analysis_sections[i] + "]", + text_colour=self._log_message_colours[1:][i % len(self._log_message_colours[1:])], + ) + + record.msg = " ".join([immediate_child_analysis_section, *subchild_analysis_sections, final_message]) + logger.handle(record) + + def _handle_exception(self, event): + """Raise the exception from the child. + + :param dict event: + :raise Exception: + :return None: + """ + exception_message = "\n\n".join( + ( + event["exception_message"], + f"The following traceback was captured from the remote service {self.child_sruid!r}:", + "".join(event["exception_traceback"]), + ) + ) + + try: + exception_type = EXCEPTIONS_MAPPING[event["exception_type"]] + + # Allow unknown exception types to still be raised. + except KeyError: + exception_type = type(event["exception_type"], (Exception,), {}) + + raise exception_type(exception_message) + + def _handle_result(self, event): + """Extract any output values and output manifest from the result, deserialising the manifest if present. + + :param dict event: + :return dict: + """ + logger.info("%r: Received an answer to question %r.", self.recipient, self.question_uuid) + + if event.get("output_manifest"): + output_manifest = Manifest.deserialise(event["output_manifest"]) + else: + output_manifest = None + + return {"output_values": event.get("output_values"), "output_manifest": output_manifest} diff --git a/octue/cloud/events/replayer.py b/octue/cloud/events/replayer.py new file mode 100644 index 000000000..203922b46 --- /dev/null +++ b/octue/cloud/events/replayer.py @@ -0,0 +1,63 @@ +import logging + +from octue.cloud.events.handler import AbstractEventHandler +from octue.cloud.events.validation import SERVICE_COMMUNICATION_SCHEMA +from octue.cloud.pub_sub.service import Service +from octue.resources.service_backends import ServiceBackend + + +logger = logging.getLogger(__name__) + + +class EventReplayer(AbstractEventHandler): + """A replayer for events retrieved asynchronously from storage. Missing events are immediately skipped. + + :param octue.cloud.pub_sub.service.Service recipient: the `Service` instance that's receiving the events + :param callable|None handle_monitor_message: a function to handle monitor messages (e.g. send them to an endpoint for plotting or displaying) - this function should take a single JSON-compatible python primitive + :param bool record_events: if `True`, record received events in the `received_events` attribute + :param dict|None event_handlers: a mapping of event type names to callables that handle each type of event. The handlers must not mutate the events. + :param dict|str schema: the JSON schema to validate events against + :param bool only_handle_result: if `True`, skip non-result events and only handle the "result" event if present + :return None: + """ + + def __init__( + self, + recipient=None, + handle_monitor_message=None, + record_events=True, + event_handlers=None, + schema=SERVICE_COMMUNICATION_SCHEMA, + only_handle_result=False, + ): + super().__init__( + recipient or Service(backend=ServiceBackend(), service_id="local/local:local"), + handle_monitor_message=handle_monitor_message, + record_events=record_events, + event_handlers=event_handlers, + schema=schema, + skip_missing_events_after=0, + only_handle_result=only_handle_result, + ) + + def handle_events(self, events): + """Handle the given events and return a handled "result" event if one is present. + + :param iter(dict) events: the events to handle + :return dict|None: the handled "result" event if present + """ + super().handle_events() + + for event in events: + self._extract_and_enqueue_event(event) + + return self._attempt_to_handle_waiting_events() + + def _extract_event_and_attributes(self, container): + """Extract an event and its attributes from the event container. + + :param dict container: the container of the event + :return (any, dict): the event and its attributes + """ + container["attributes"]["order"] = int(container["attributes"]["order"]) + return container["event"], container["attributes"] diff --git a/octue/cloud/validation.py b/octue/cloud/events/validation.py similarity index 80% rename from octue/cloud/validation.py rename to octue/cloud/events/validation.py index 98137651e..422b42163 100644 --- a/octue/cloud/validation.py +++ b/octue/cloud/events/validation.py @@ -6,24 +6,26 @@ from octue.compatibility import warn_if_incompatible -logger = logging.getLogger(__name__) +VALID_EVENT_KINDS = {"delivery_acknowledgement", "heartbeat", "log_record", "monitor_message", "exception", "result"} -SERVICE_COMMUNICATION_SCHEMA = {"$ref": "https://jsonschema.registry.octue.com/octue/service-communication/0.8.2.json"} +SERVICE_COMMUNICATION_SCHEMA = {"$ref": "https://jsonschema.registry.octue.com/octue/service-communication/0.10.0.json"} SERVICE_COMMUNICATION_SCHEMA_INFO_URL = "https://strands.octue.com/octue/service-communication" SERVICE_COMMUNICATION_SCHEMA_VERSION = os.path.splitext(SERVICE_COMMUNICATION_SCHEMA["$ref"])[0].split("/")[-1] -# Instantiate a JSON schema validator to cache the service communication schema. This avoids getting it from the +# Instantiate a JSON schema validator to cache the service communication schema. This avoids downloading it from the # registry every time a message is validated against it. jsonschema.Draft202012Validator.check_schema(SERVICE_COMMUNICATION_SCHEMA) jsonschema_validator = jsonschema.Draft202012Validator(SERVICE_COMMUNICATION_SCHEMA) +logger = logging.getLogger(__name__) + -def is_event_valid(event, attributes, receiving_service, parent_sdk_version, child_sdk_version, schema=None): +def is_event_valid(event, attributes, recipient, parent_sdk_version, child_sdk_version, schema=None): """Check if the event and its attributes are valid according to the Octue services communication schema. :param dict event: the event to validate :param dict attributes: the attributes of the event to validate - :param octue.cloud.pub_sub.service.Service receiving_service: the service receiving and validating the event + :param octue.cloud.pub_sub.service.Service recipient: the service receiving and validating the event :param str parent_sdk_version: the semantic version of Octue SDK running on the parent :param str child_sdk_version: the semantic version of Octue SDK running on the child :param dict|None schema: the schema to validate the event and its attributes against; if `None`, this defaults to the service communication schema used in this version of Octue SDK @@ -33,7 +35,7 @@ def is_event_valid(event, attributes, receiving_service, parent_sdk_version, chi raise_if_event_is_invalid( event, attributes, - receiving_service, + recipient, parent_sdk_version, child_sdk_version, schema=schema, @@ -44,25 +46,19 @@ def is_event_valid(event, attributes, receiving_service, parent_sdk_version, chi return True -def raise_if_event_is_invalid( - event, - attributes, - receiving_service, - parent_sdk_version, - child_sdk_version, - schema=None, -): +def raise_if_event_is_invalid(event, attributes, recipient, parent_sdk_version, child_sdk_version, schema=None): """Raise an error if the event or its attributes aren't valid according to the Octue services communication schema. :param dict event: the event to validate :param dict attributes: the attributes of the event to validate - :param octue.cloud.pub_sub.service.Service receiving_service: the service receiving and validating the event + :param octue.cloud.pub_sub.service.Service recipient: the service receiving and validating the event :param str parent_sdk_version: the semantic version of Octue SDK running on the parent :param str child_sdk_version: the semantic version of Octue SDK running on the child :param dict|None schema: the schema to validate the event and its attributes against; if `None`, this defaults to the service communication schema used in this version of Octue SDK :raise jsonschema.ValidationError: if the event or its attributes are invalid :return None: """ + # Transform attributes to a dictionary in the case they're a different kind of mapping. data = {"event": event, "attributes": dict(attributes)} if schema is None: @@ -82,7 +78,7 @@ def raise_if_event_is_invalid( logger.exception( "%r received an event that doesn't conform with version %s of the service communication schema (%s): %r.", - receiving_service, + recipient, SERVICE_COMMUNICATION_SCHEMA_VERSION, SERVICE_COMMUNICATION_SCHEMA_INFO_URL, event, diff --git a/octue/cloud/pub_sub/__init__.py b/octue/cloud/pub_sub/__init__.py index bdb8ca04f..b6239757a 100644 --- a/octue/cloud/pub_sub/__init__.py +++ b/octue/cloud/pub_sub/__init__.py @@ -1,5 +1,41 @@ +from octue.cloud.events import OCTUE_SERVICES_PREFIX +from octue.cloud.service_id import convert_service_id_to_pub_sub_form + from .subscription import Subscription from .topic import Topic __all__ = ["Subscription", "Topic"] + + +def create_push_subscription( + project_name, + sruid, + push_endpoint, + subscription_filter=None, + expiration_time=None, +): + """Create a Google Pub/Sub push subscription for an Octue service for it to receive questions from parents. If a + corresponding topic doesn't exist, it will be created first. + + :param str project_name: the name of the Google Cloud project in which the subscription will be created + :param str sruid: the SRUID (service revision unique identifier) + :param str push_endpoint: the HTTP/HTTPS endpoint of the service to push to. It should be fully formed and include the 'https://' prefix + :param str|None subscription_filter: if specified, the filter to apply to the subscription; otherwise, no filter is applied + :param float|None expiration_time: the number of seconds of inactivity after which the subscription should expire. If not provided, no expiration time is applied to the subscription + :return None: + """ + if expiration_time: + expiration_time = float(expiration_time) + else: + expiration_time = None + + subscription = Subscription( + name=convert_service_id_to_pub_sub_form(sruid), + topic=Topic(name=OCTUE_SERVICES_PREFIX, project_name=project_name), + filter=subscription_filter, + expiration_time=expiration_time, + push_endpoint=push_endpoint, + ) + + subscription.create() diff --git a/octue/cloud/pub_sub/bigquery.py b/octue/cloud/pub_sub/bigquery.py new file mode 100644 index 000000000..d02833c07 --- /dev/null +++ b/octue/cloud/pub_sub/bigquery.py @@ -0,0 +1,89 @@ +import json + +from google.cloud.bigquery import Client, QueryJobConfig, ScalarQueryParameter + +from octue.cloud.events.validation import VALID_EVENT_KINDS + + +def get_events( + table_id, + sender, + question_uuid, + kind=None, + include_attributes=False, + include_backend_metadata=False, + limit=1000, +): + """Get Octue service events for a question from a sender from a Google BigQuery event store. + + :param str table_id: the full ID of the table e.g. "your-project.your-dataset.your-table" + :param str sender: the SRUID of the sender of the events + :param str question_uuid: the UUID of the question to get the events for + :param str|None kind: the kind of event to get; if `None`, all event kinds are returned + :param bool include_attributes: if `True`, include events' attributes (excluding question UUID) + :param bool include_backend_metadata: if `True`, include the service backend metadata + :param int limit: the maximum number of events to return + :return list(dict): the events for the question + """ + if kind: + if kind not in VALID_EVENT_KINDS: + raise ValueError(f"`kind` must be one of {VALID_EVENT_KINDS!r}; received {kind!r}.") + + event_kind_condition = [f'AND JSON_EXTRACT_SCALAR(event, "$.kind") = "{kind}"'] + else: + event_kind_condition = [] + + client = Client() + fields = ["`event`"] + + if include_attributes: + fields.extend( + ( + "`datetime`", + "`uuid`", + "`originator`", + "`sender`", + "`sender_type`", + "`sender_sdk_version`", + "`recipient`", + "`order`", + "`other_attributes`", + ) + ) + + if include_backend_metadata: + fields.extend(("`backend`", "`backend_metadata`")) + + query = "\n".join( + [ + f"SELECT {', '.join(fields)} FROM `{table_id}`", + "WHERE sender=@sender", + "AND question_uuid=@question_uuid", + *event_kind_condition, + "ORDER BY `order`", + "LIMIT @limit", + ] + ) + + job_config = QueryJobConfig( + query_parameters=[ + ScalarQueryParameter("sender", "STRING", sender), + ScalarQueryParameter("question_uuid", "STRING", question_uuid), + ScalarQueryParameter("limit", "INTEGER", limit), + ] + ) + + query_job = client.query(query, job_config=job_config) + rows = query_job.result() + df = rows.to_dataframe() + + # Convert JSON strings to python primitives. + df["event"] = df["event"].map(json.loads) + + if "other_attributes" in df: + df["other_attributes"] = df["other_attributes"].map(json.loads) + + if "backend_metadata" in df: + df["backend_metadata"] = df["backend_metadata"].map(json.loads) + + return df.to_dict(orient="records") diff --git a/octue/cloud/pub_sub/events.py b/octue/cloud/pub_sub/events.py index d61af3876..9d0b700b8 100644 --- a/octue/cloud/pub_sub/events.py +++ b/octue/cloud/pub_sub/events.py @@ -1,11 +1,25 @@ import base64 import json +import logging +import time +from datetime import datetime, timedelta +from google.api_core import retry +from google.cloud.pubsub_v1 import SubscriberClient + +from octue.cloud.events.handler import AbstractEventHandler +from octue.cloud.events.validation import SERVICE_COMMUNICATION_SCHEMA from octue.utils.decoders import OctueJSONDecoder from octue.utils.objects import getattr_or_subscribe +from octue.utils.threads import RepeatingTimer + + +logger = logging.getLogger(__name__) +MAX_SIMULTANEOUS_MESSAGES_PULL = 50 -def extract_event_and_attributes_from_pub_sub(message): + +def extract_event_and_attributes_from_pub_sub_message(message): """Extract an Octue service event and its attributes from a Google Pub/Sub message in either direct Pub/Sub format or in the Google Cloud Run format. @@ -15,18 +29,18 @@ def extract_event_and_attributes_from_pub_sub(message): # Cast attributes to a dictionary to avoid defaultdict-like behaviour from Pub/Sub message attributes container. attributes = dict(getattr_or_subscribe(message, "attributes")) - converted_attributes = { - "sender_type": attributes["sender_type"], - "question_uuid": attributes["question_uuid"], - "message_number": int(attributes["message_number"]), - "version": attributes["version"], - } + # Deserialise the `order` and `forward_logs` fields if they're present (don't assume they are before validation). + if attributes.get("order"): + attributes["order"] = int(attributes["order"]) - if "forward_logs" in attributes: - converted_attributes["forward_logs"] = bool(int(attributes["forward_logs"])) + # Required for question events. + if attributes.get("sender_type") == "PARENT": + forward_logs = attributes.get("forward_logs") - if "save_diagnostics" in attributes: - converted_attributes["save_diagnostics"] = attributes["save_diagnostics"] + if forward_logs: + attributes["forward_logs"] = bool(int(forward_logs)) + else: + attributes["forward_logs"] = None try: # Parse event directly from Pub/Sub or Dataflow. @@ -35,4 +49,191 @@ def extract_event_and_attributes_from_pub_sub(message): # Parse event from Google Cloud Run. event = json.loads(base64.b64decode(message["data"]).decode("utf-8").strip(), cls=OctueJSONDecoder) - return event, converted_attributes + return event, attributes + + +class GoogleCloudPubSubEventHandler(AbstractEventHandler): + """A synchronous handler for events received as Google Pub/Sub messages from a pull subscription. + + :param octue.cloud.pub_sub.subscription.Subscription subscription: the subscription messages are pulled from + :param octue.cloud.pub_sub.service.Service recipient: the `Service` instance that's receiving the events + :param callable|None handle_monitor_message: a function to handle monitor messages (e.g. send them to an endpoint for plotting or displaying) - this function should take a single JSON-compatible python primitive + :param bool record_events: if `True`, record received events in the `received_events` attribute + :param dict|None event_handlers: a mapping of event type names to callables that handle each type of event. The handlers must not mutate the events. + :param dict|str schema: the JSON schema to validate events against + :param int|float skip_missing_events_after: the number of seconds after which to skip any events if they haven't arrived but subsequent events have + :return None: + """ + + def __init__( + self, + subscription, + recipient, + handle_monitor_message=None, + record_events=True, + event_handlers=None, + schema=SERVICE_COMMUNICATION_SCHEMA, + skip_missing_events_after=10, + ): + self.subscription = subscription + + super().__init__( + recipient, + handle_monitor_message=handle_monitor_message, + record_events=record_events, + event_handlers=event_handlers, + schema=schema, + skip_missing_events_after=skip_missing_events_after, + ) + + self._subscriber = SubscriberClient() + self._heartbeat_checker = None + self._last_heartbeat = None + self._alive = True + + @property + def total_run_time(self): + """The amount of time elapsed since `self.handle_events` was called. If it hasn't been called yet, this is + `None`. + + :return float|None: the amount of time [s] since `self.handle_events` was called + """ + if self._start_time is None: + return None + + return time.perf_counter() - self._start_time + + @property + def _time_since_last_heartbeat(self): + """The amount of time since the last heartbeat was received. If no heartbeat has been received, this is `None`. + + :return datetime.timedelta|None: + """ + if not self._last_heartbeat: + return None + + return datetime.now() - self._last_heartbeat + + def handle_events(self, timeout=60, maximum_heartbeat_interval=300): + """Pull events from the subscription and handle them in the order they were sent until a "result" event is + handled, then return the handled result. + + :param float|None timeout: how long to wait for an answer before raising a `TimeoutError` + :param int|float maximum_heartbeat_interval: the maximum amount of time [s] allowed between child heartbeats before an error is raised + :raise TimeoutError: if the timeout is exceeded before receiving the final event + :return dict: the handled "result" event + """ + super().handle_events() + + self._heartbeat_checker = RepeatingTimer( + interval=maximum_heartbeat_interval, + function=self._monitor_heartbeat, + kwargs={"maximum_heartbeat_interval": maximum_heartbeat_interval}, + ) + + try: + self._heartbeat_checker.daemon = True + self._heartbeat_checker.start() + + while self._alive: + pull_timeout = self._check_timeout_and_get_pull_timeout(timeout) + self._pull_and_enqueue_available_events(timeout=pull_timeout) + result = self._attempt_to_handle_waiting_events() + + if result is not None: + return result + + finally: + self._heartbeat_checker.cancel() + self._subscriber.close() + + raise TimeoutError( + f"No heartbeat has been received within the maximum allowed interval of {maximum_heartbeat_interval}s." + ) + + def _monitor_heartbeat(self, maximum_heartbeat_interval): + """Change the alive status to `False` and cancel the heartbeat checker if a heartbeat hasn't been received + within the maximum allowed time interval since the last received heartbeat. + + :param float|int maximum_heartbeat_interval: the maximum amount of time [s] allowed between child heartbeats without raising an error + :return None: + """ + maximum_heartbeat_interval = timedelta(seconds=maximum_heartbeat_interval) + + if self._last_heartbeat and self._time_since_last_heartbeat <= maximum_heartbeat_interval: + self._alive = True + return + + self._alive = False + self._heartbeat_checker.cancel() + + def _check_timeout_and_get_pull_timeout(self, timeout): + """Check if the message handling timeout has been exceeded and, if it hasn't, calculate and return the timeout + for the next message pull. If the timeout has been exceeded, raise an error. + + :param int|float|None timeout: the timeout [s] for handling all messages, or `None` if there's no timeout + :raise TimeoutError: if the timeout has been exceeded + :return int|float|None: the timeout for the next message pull [s], or `None` if there's no timeout + """ + if timeout is None: + return None + + # Get the total run time once in case it's very close to the timeout - this rules out a negative pull timeout + # being returned below. + total_run_time = self.total_run_time + + if total_run_time > timeout: + raise TimeoutError(f"No final result received from {self.subscription.topic!r} after {timeout} seconds.") + + return timeout - total_run_time + + def _pull_and_enqueue_available_events(self, timeout): + """Pull as many events from the subscription as are available and enqueue them in `self.waiting_events`, + raising a `TimeoutError` if the timeout is exceeded before succeeding. + + :param float|None timeout: how long to wait for the event [s] before raising a `TimeoutError` + :raise TimeoutError|concurrent.futures.TimeoutError: if the timeout is exceeded + :return None: + """ + pull_start_time = time.perf_counter() + attempt = 1 + + while self._alive: + logger.debug("Pulling events from Google Pub/Sub: attempt %d.", attempt) + + pull_response = self._subscriber.pull( + request={"subscription": self.subscription.path, "max_messages": MAX_SIMULTANEOUS_MESSAGES_PULL}, + retry=retry.Retry(), + ) + + if len(pull_response.received_messages) > 0: + break + else: + logger.debug("Google Pub/Sub pull response timed out early.") + attempt += 1 + + pull_run_time = time.perf_counter() - pull_start_time + + if timeout is not None and pull_run_time > timeout: + raise TimeoutError(f"No message received from {self.subscription.topic!r} after {timeout} seconds.") + + if not pull_response.received_messages: + return + + self._subscriber.acknowledge( + request={ + "subscription": self.subscription.path, + "ack_ids": [message.ack_id for message in pull_response.received_messages], + } + ) + + for event in pull_response.received_messages: + self._extract_and_enqueue_event(event) + + def _extract_event_and_attributes(self, container): + """Extract an event and its attributes from a Pub/Sub message. + + :param dict container: a Pub/Sub message + :return (any, dict): the event and its attributes + """ + return extract_event_and_attributes_from_pub_sub_message(container.message) diff --git a/octue/cloud/pub_sub/logging.py b/octue/cloud/pub_sub/logging.py index 286269cd0..e8d78a6a8 100644 --- a/octue/cloud/pub_sub/logging.py +++ b/octue/cloud/pub_sub/logging.py @@ -5,22 +5,26 @@ ANSI_ESCAPE_SEQUENCES_PATTERN = r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])" -class GooglePubSubHandler(logging.Handler): +class GoogleCloudPubSubHandler(logging.Handler): """A log handler that publishes log records to a Google Cloud Pub/Sub topic. - :param callable message_sender: the `_send_message` method of the service that instantiated this instance - :param octue.cloud.pub_sub.topic.Topic topic: topic to publish log records to + :param callable event_emitter: the `_emit_event` method of the service that instantiated this instance :param str question_uuid: the UUID of the question to handle log records for + :param str originator: the SRUID of the service that asked the question these log records are related to + :param str recipient: the SRUID of the service to send these log records to + :param octue.cloud.events.counter.EventCounter order: an event counter keeping track of the order of emitted events :param float timeout: timeout in seconds for attempting to publish each log record :return None: """ - def __init__(self, message_sender, topic, question_uuid, timeout=60, *args, **kwargs): + def __init__(self, event_emitter, question_uuid, originator, recipient, order, timeout=60, *args, **kwargs): super().__init__(*args, **kwargs) - self.topic = topic self.question_uuid = question_uuid + self.originator = originator + self.recipient = recipient + self.order = order self.timeout = timeout - self._send_message = message_sender + self._emit_event = event_emitter def emit(self, record): """Serialise the log record as a dictionary and publish it to the topic. @@ -29,12 +33,14 @@ def emit(self, record): :return None: """ try: - self._send_message( + self._emit_event( { "kind": "log_record", "log_record": self._convert_log_record_to_primitives(record), }, - topic=self.topic, + originator=self.originator, + recipient=self.recipient, + order=self.order, attributes={ "question_uuid": self.question_uuid, "sender_type": "CHILD", # The sender type is repeated here as a string to avoid a circular import. diff --git a/octue/cloud/pub_sub/message_handler.py b/octue/cloud/pub_sub/message_handler.py deleted file mode 100644 index 296d35e47..000000000 --- a/octue/cloud/pub_sub/message_handler.py +++ /dev/null @@ -1,458 +0,0 @@ -import importlib.metadata -import logging -import math -import os -import re -import time -from datetime import datetime, timedelta - -from google.api_core import retry -from google.cloud.pubsub_v1 import SubscriberClient - -from octue.cloud import EXCEPTIONS_MAPPING -from octue.cloud.pub_sub.events import extract_event_and_attributes_from_pub_sub -from octue.cloud.validation import SERVICE_COMMUNICATION_SCHEMA, is_event_valid -from octue.definitions import GOOGLE_COMPUTE_PROVIDERS -from octue.log_handlers import COLOUR_PALETTE -from octue.resources.manifest import Manifest -from octue.utils.threads import RepeatingTimer - - -if os.environ.get("COMPUTE_PROVIDER", "UNKNOWN") in GOOGLE_COMPUTE_PROVIDERS: - # Google Cloud logs don't support colour currently - provide a no-operation function. - colourise = lambda string, text_colour=None, background_colour=None: string -else: - from octue.utils.colour import colourise - -logger = logging.getLogger(__name__) - - -MAX_SIMULTANEOUS_MESSAGES_PULL = 50 -PARENT_SDK_VERSION = importlib.metadata.version("octue") - - -class OrderedMessageHandler: - """A handler for Google Pub/Sub messages received via a pull subscription that ensures messages are handled in the - order they were sent. - - :param octue.cloud.pub_sub.subscription.Subscription subscription: the subscription messages are pulled from - :param octue.cloud.pub_sub.service.Service receiving_service: the service that's receiving the messages - :param callable|None handle_monitor_message: a function to handle monitor messages (e.g. send them to an endpoint for plotting or displaying) - this function should take a single JSON-compatible python primitive - :param bool record_messages: if `True`, record received messages in the `received_messages` attribute - :param str service_name: an arbitrary name to refer to the service subscribed to by (used for labelling its remote log messages) - :param dict|None message_handlers: a mapping of message type names to callables that handle each type of message. The handlers should not mutate the messages. - :param dict|str schema: the JSON schema (or URI of one) to validate messages against - :param int|float skip_missing_messages_after: the number of seconds after which to skip any messages if they haven't arrived but subsequent messages have - :return None: - """ - - def __init__( - self, - subscription, - receiving_service, - handle_monitor_message=None, - record_messages=True, - service_name="REMOTE", - message_handlers=None, - schema=SERVICE_COMMUNICATION_SCHEMA, - skip_missing_messages_after=10, - ): - self.subscription = subscription - self.receiving_service = receiving_service - self.handle_monitor_message = handle_monitor_message - self.record_messages = record_messages - self.service_name = service_name - self.schema = schema - - self.skip_missing_messages_after = skip_missing_messages_after - self._missing_message_detection_time = None - - self.question_uuid = self.subscription.path.split(".")[-1] - self.handled_messages = [] - self.waiting_messages = None - self._subscriber = SubscriberClient() - self._child_sdk_version = None - self._heartbeat_checker = None - self._last_heartbeat = None - self._alive = True - self._start_time = None - self._previous_message_number = -1 - self._earliest_waiting_message_number = math.inf - - self._message_handlers = message_handlers or { - "delivery_acknowledgement": self._handle_delivery_acknowledgement, - "heartbeat": self._handle_heartbeat, - "monitor_message": self._handle_monitor_message, - "log_record": self._handle_log_message, - "exception": self._handle_exception, - "result": self._handle_result, - } - - self._log_message_colours = [COLOUR_PALETTE[1], *COLOUR_PALETTE[3:]] - - @property - def total_run_time(self): - """Get the amount of time elapsed since `self.handle_messages` was called. If it hasn't been called yet, it will - be `None`. - - :return float|None: the amount of time since `self.handle_messages` was called (in seconds) - """ - if self._start_time is None: - return None - - return time.perf_counter() - self._start_time - - @property - def time_since_missing_message(self): - """Get the amount of time elapsed since the last missing message was detected. If no missing messages have been - detected or they've already been skipped past, `None` is returned. - - :return float|None: - """ - if self._missing_message_detection_time is None: - return None - - return time.perf_counter() - self._missing_message_detection_time - - @property - def _time_since_last_heartbeat(self): - """Get the time period since the last heartbeat was received. - - :return datetime.timedelta|None: - """ - if not self._last_heartbeat: - return None - - return datetime.now() - self._last_heartbeat - - def handle_messages(self, timeout=60, maximum_heartbeat_interval=300): - """Pull messages and handle them in the order they were sent until a result is returned by a message handler, - then return that result. - - :param float|None timeout: how long to wait for an answer before raising a `TimeoutError` - :param int|float maximum_heartbeat_interval: the maximum amount of time (in seconds) allowed between child heartbeats before an error is raised - :raise TimeoutError: if the timeout is exceeded before receiving the final message - :return dict: the first result returned by a message handler - """ - self._start_time = time.perf_counter() - self.waiting_messages = {} - self._previous_message_number = -1 - - self._heartbeat_checker = RepeatingTimer( - interval=maximum_heartbeat_interval, - function=self._monitor_heartbeat, - kwargs={"maximum_heartbeat_interval": maximum_heartbeat_interval}, - ) - - try: - self._heartbeat_checker.daemon = True - self._heartbeat_checker.start() - - while self._alive: - pull_timeout = self._check_timeout_and_get_pull_timeout(timeout) - self._pull_and_enqueue_available_messages(timeout=pull_timeout) - result = self._attempt_to_handle_waiting_messages() - - if result is not None: - return result - - finally: - self._heartbeat_checker.cancel() - self._subscriber.close() - - raise TimeoutError( - f"No heartbeat has been received within the maximum allowed interval of {maximum_heartbeat_interval}s." - ) - - def _monitor_heartbeat(self, maximum_heartbeat_interval): - """Change the alive status to `False` and cancel the heartbeat checker if a heartbeat hasn't been received - within the maximum allowed time interval measured from the moment of calling. - - :param float|int maximum_heartbeat_interval: the maximum amount of time (in seconds) allowed between child heartbeats without raising an error - :return None: - """ - maximum_heartbeat_interval = timedelta(seconds=maximum_heartbeat_interval) - - if self._last_heartbeat and self._time_since_last_heartbeat <= maximum_heartbeat_interval: - self._alive = True - return - - self._alive = False - self._heartbeat_checker.cancel() - - def _check_timeout_and_get_pull_timeout(self, timeout): - """Check if the timeout has been exceeded and, if it hasn't, return the timeout for the next message pull. If - the timeout has been exceeded, raise an error. - - :param int|float|None timeout: the timeout for handling all messages - :raise TimeoutError: if the timeout has been exceeded - :return int|float: the timeout for the next message pull in seconds - """ - if timeout is None: - return None - - # Get the total run time once in case it's very close to the timeout - this rules out a negative pull timeout - # being returned below. - total_run_time = self.total_run_time - - if total_run_time > timeout: - raise TimeoutError( - f"No final answer received from topic {self.subscription.topic.path!r} after {timeout} seconds." - ) - - return timeout - total_run_time - - def _pull_and_enqueue_available_messages(self, timeout): - """Pull as many messages from the subscription as are available and enqueue them in `self.waiting_messages`, - raising a `TimeoutError` if the timeout is exceeded before succeeding. - - :param float|None timeout: how long to wait in seconds for the message before raising a `TimeoutError` - :raise TimeoutError|concurrent.futures.TimeoutError: if the timeout is exceeded - :return None: - """ - pull_start_time = time.perf_counter() - attempt = 1 - - while self._alive: - logger.debug("Pulling messages from Google Pub/Sub: attempt %d.", attempt) - - pull_response = self._subscriber.pull( - request={"subscription": self.subscription.path, "max_messages": MAX_SIMULTANEOUS_MESSAGES_PULL}, - retry=retry.Retry(), - ) - - if len(pull_response.received_messages) > 0: - break - else: - logger.debug("Google Pub/Sub pull response timed out early.") - attempt += 1 - - pull_run_time = time.perf_counter() - pull_start_time - - if timeout is not None and pull_run_time > timeout: - raise TimeoutError( - f"No message received from topic {self.subscription.topic.path!r} after {timeout} seconds.", - ) - - if not pull_response.received_messages: - return - - self._subscriber.acknowledge( - request={ - "subscription": self.subscription.path, - "ack_ids": [message.ack_id for message in pull_response.received_messages], - } - ) - - for message in pull_response.received_messages: - self._extract_and_enqueue_event(message) - - self._earliest_waiting_message_number = min(self.waiting_messages.keys()) - - def _extract_and_enqueue_event(self, message): - """Extract an event from the Pub/Sub message and add it to `self.waiting_messages`. - - :param dict message: - :return None: - """ - logger.debug("%r received a message related to question %r.", self.receiving_service, self.question_uuid) - event, attributes = extract_event_and_attributes_from_pub_sub(message.message) - - if not is_event_valid( - event=event, - attributes=attributes, - receiving_service=self.receiving_service, - parent_sdk_version=PARENT_SDK_VERSION, - child_sdk_version=attributes.get("version"), - schema=self.schema, - ): - return - - # Get the child's Octue SDK version from the first message. - if not self._child_sdk_version: - self._child_sdk_version = attributes["version"] - - message_number = attributes["message_number"] - - if message_number in self.waiting_messages: - logger.warning( - "%r: Message with duplicate message number %d received for question %s - overwriting original message.", - self.receiving_service, - message_number, - self.question_uuid, - ) - - self.waiting_messages[message_number] = event - - def _attempt_to_handle_waiting_messages(self): - """Attempt to handle messages waiting in `self.waiting_messages`. If these messages aren't consecutive to the - last handled message (i.e. if messages have been received out of order and the next in-order message hasn't been - received yet), just return. After the missing message wait time has passed, if this set of missing messages - haven't arrived but subsequent ones have, skip to the earliest waiting message and continue from there. - - :return any|None: either a non-`None` result from a message handler or `None` if nothing was returned by the message handlers or if the next in-order message hasn't been received yet - """ - while self.waiting_messages: - try: - # If the next consecutive message has been received: - message = self.waiting_messages.pop(self._previous_message_number + 1) - - # If the next consecutive message hasn't been received: - except KeyError: - # Start the missing message timer if it isn't already running. - if self._missing_message_detection_time is None: - self._missing_message_detection_time = time.perf_counter() - - if self.time_since_missing_message > self.skip_missing_messages_after: - message = self._skip_to_earliest_waiting_message() - - # Declare there are no more missing messages. - self._missing_message_detection_time = None - - if not message: - return - - else: - return - - result = self._handle_message(message) - - if result is not None: - return result - - def _skip_to_earliest_waiting_message(self): - """Get the earliest waiting message and set the message handler up to continue from it. - - :return dict|None: - """ - try: - message = self.waiting_messages.pop(self._earliest_waiting_message_number) - except KeyError: - return - - number_of_missing_messages = self._earliest_waiting_message_number - self._previous_message_number - 1 - - # Let the message handler know it can handle the next earliest message. - self._previous_message_number = self._earliest_waiting_message_number - 1 - - logger.warning( - "%r: %d consecutive messages missing for question %r after %ds - skipping to next earliest waiting message " - "(message %d).", - self.receiving_service, - number_of_missing_messages, - self.question_uuid, - self.skip_missing_messages_after, - self._earliest_waiting_message_number, - ) - - return message - - def _handle_message(self, message): - """Pass a message to its handler and update the previous message number. - - :param dict message: - :return dict|None: - """ - self._previous_message_number += 1 - - if self.record_messages: - self.handled_messages.append(message) - - handler = self._message_handlers[message["kind"]] - return handler(message) - - def _handle_delivery_acknowledgement(self, message): - """Mark the question as delivered to prevent resending it. - - :param dict message: - :return None: - """ - logger.info("%r's question was delivered at %s.", self.receiving_service, message["datetime"]) - - def _handle_heartbeat(self, message): - """Record the time the heartbeat was received. - - :param dict message: - :return None: - """ - self._last_heartbeat = datetime.now() - logger.info("Heartbeat received from service %r for question %r.", self.service_name, self.question_uuid) - - def _handle_monitor_message(self, message): - """Send a monitor message to the handler if one has been provided. - - :param dict message: - :return None: - """ - logger.debug("%r received a monitor message.", self.receiving_service) - - if self.handle_monitor_message is not None: - self.handle_monitor_message(message["data"]) - - def _handle_log_message(self, message): - """Deserialise the message into a log record and pass it to the local log handlers, adding [] to - the start of the log message. - - :param dict message: - :return None: - """ - record = logging.makeLogRecord(message["log_record"]) - - # Add information about the immediate child sending the message and colour it with the first colour in the - # colour palette. - immediate_child_analysis_section = colourise( - f"[{self.service_name} | analysis-{self.question_uuid}]", - text_colour=self._log_message_colours[0], - ) - - # Colour any analysis sections from children of the immediate child with the rest of the colour palette. - subchild_analysis_sections = [section.strip("[") for section in re.split("] ", record.msg)] - final_message = subchild_analysis_sections.pop(-1) - - for i in range(len(subchild_analysis_sections)): - subchild_analysis_sections[i] = colourise( - "[" + subchild_analysis_sections[i] + "]", - text_colour=self._log_message_colours[1:][i % len(self._log_message_colours[1:])], - ) - - record.msg = " ".join([immediate_child_analysis_section, *subchild_analysis_sections, final_message]) - logger.handle(record) - - def _handle_exception(self, message): - """Raise the exception from the responding service that is serialised in `data`. - - :param dict message: - :raise Exception: - :return None: - """ - exception_message = "\n\n".join( - ( - message["exception_message"], - f"The following traceback was captured from the remote service {self.service_name!r}:", - "".join(message["exception_traceback"]), - ) - ) - - try: - exception_type = EXCEPTIONS_MAPPING[message["exception_type"]] - - # Allow unknown exception types to still be raised. - except KeyError: - exception_type = type(message["exception_type"], (Exception,), {}) - - raise exception_type(exception_message) - - def _handle_result(self, message): - """Convert the result to the correct form, deserialising the output manifest if it is present in the message. - - :param dict message: - :return dict: - """ - logger.info("%r received an answer to question %r.", self.receiving_service, self.question_uuid) - - if message.get("output_manifest"): - output_manifest = Manifest.deserialise(message["output_manifest"]) - else: - output_manifest = None - - return {"output_values": message.get("output_values"), "output_manifest": output_manifest} diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py index 5f6c27f70..054d8c255 100644 --- a/octue/cloud/pub_sub/service.py +++ b/octue/cloud/pub_sub/service.py @@ -14,21 +14,22 @@ from google.cloud import pubsub_v1 import octue.exceptions +from octue.cloud.events import OCTUE_SERVICES_PREFIX +from octue.cloud.events.counter import EventCounter +from octue.cloud.events.validation import raise_if_event_is_invalid from octue.cloud.pub_sub import Subscription, Topic -from octue.cloud.pub_sub.events import extract_event_and_attributes_from_pub_sub -from octue.cloud.pub_sub.logging import GooglePubSubHandler -from octue.cloud.pub_sub.message_handler import OrderedMessageHandler +from octue.cloud.pub_sub.events import GoogleCloudPubSubEventHandler, extract_event_and_attributes_from_pub_sub_message +from octue.cloud.pub_sub.logging import GoogleCloudPubSubHandler from octue.cloud.service_id import ( convert_service_id_to_pub_sub_form, create_sruid, get_default_sruid, - get_sruid_from_pub_sub_resource_name, raise_if_revision_not_registered, split_service_id, validate_sruid, ) -from octue.cloud.validation import raise_if_event_is_invalid from octue.compatibility import warn_if_incompatible +from octue.utils.dictionaries import make_minimal_dictionary from octue.utils.encoders import OctueJSONEncoder from octue.utils.exceptions import convert_exception_to_primitives from octue.utils.threads import RepeatingTimer @@ -36,10 +37,10 @@ logger = logging.getLogger(__name__) -# A lock to ensure only one message can be sent at a time so that the message number is incremented correctly when -# messages are being sent on multiple threads (e.g. via the main thread and a periodic monitor message thread). This -# avoids 1) messages overwriting each other in the parent's message handler and 2) messages losing their order. -send_message_lock = threading.Lock() +# A lock to ensure only one event can be emitted at a time so that the order is incremented correctly when events are +# being emitted on multiple threads (e.g. via the main thread and a periodic monitor message thread). This avoids 1) +# events overwriting each other in the parent's message handler and 2) events losing their order. +emit_event_lock = threading.Lock() DEFAULT_NAMESPACE = "default" ANSWERS_NAMESPACE = "answers" @@ -86,13 +87,13 @@ def __init__(self, backend, service_id=None, run_function=None, name=None, servi self.backend = backend self.run_function = run_function self.name = name - self.service_registries = service_registries self._pub_sub_id = convert_service_id_to_pub_sub_form(self.id) self._local_sdk_version = importlib.metadata.version("octue") self._publisher = None - self._message_handler = None + self._services_topic = None + self._event_handler = None def __repr__(self): """Represent the service as a string. @@ -115,15 +116,34 @@ def publisher(self): return self._publisher @property - def received_messages(self): - """Get the messages received by the service from a child service while running the `wait_for_answer` method. If - the `wait_for_answer` method hasn't been run, `None` is returned. If an empty list is returned, no messages have - been received. + def services_topic(self): + """Get the Octue services topic that all events in the project are published to. No topic is instantiated until + this property is called for the first time. This allows checking for the `GOOGLE_APPLICATION_CREDENTIALS` + environment variable to be put off until it's needed. + + :raise octue.exceptions.ServiceNotFound: if the topic doesn't exist in the project + :return octue.cloud.pub_sub.topic.Topic: the Octue services topic for the project + """ + if not self._services_topic: + topic = Topic(name=OCTUE_SERVICES_PREFIX, project_name=self.backend.project_name) + + if not topic.exists(): + raise octue.exceptions.ServiceNotFound(f"{topic!r} cannot be found.") + + self._services_topic = topic + + return self._services_topic + + @property + def received_events(self): + """Get the events received from a child service while running the `wait_for_answer` method. If the + `wait_for_answer` method hasn't been run, `None` is returned. If an empty list is returned, no events have been + received. :return list(dict)|None: """ - if self._message_handler: - return self._message_handler.handled_messages + if self._event_handler: + return self._event_handler.handled_events return None def serve(self, timeout=None, delete_topic_and_subscription_on_exit=False, allow_existing=False, detach=False): @@ -137,18 +157,15 @@ def serve(self, timeout=None, delete_topic_and_subscription_on_exit=False, allow :return (google.cloud.pubsub_v1.subscriber.futures.StreamingPullFuture, google.cloud.pubsub_v1.SubscriberClient): """ logger.info("Starting %r.", self) - topic = Topic(name=self._pub_sub_id, project_name=self.backend.project_name) subscription = Subscription( - name=self._pub_sub_id, - topic=topic, - project_name=self.backend.project_name, - filter=f'attributes.sender_type = "{PARENT_SENDER_TYPE}"', + name=".".join((OCTUE_SERVICES_PREFIX, self._pub_sub_id)), + topic=self.services_topic, + filter=f'attributes.recipient = "{self.id}" AND attributes.sender_type = "{PARENT_SENDER_TYPE}"', expiration_time=None, ) try: - topic.create(allow_existing=allow_existing) subscription.create(allow_existing=allow_existing) except google.api_core.exceptions.AlreadyExists: raise octue.exceptions.ServiceAlreadyExists(f"A service with the ID {self.id!r} already exists.") @@ -179,11 +196,8 @@ def serve(self, timeout=None, delete_topic_and_subscription_on_exit=False, allow if subscription.creation_triggered_locally: subscription.delete() - if topic.creation_triggered_locally: - topic.delete() - except Exception: - logger.error("Deletion of topic and/or subscription %r failed.", topic.name) + logger.error("Deletion of %r failed.", subscription) subscriber.close() @@ -206,30 +220,33 @@ def answer(self, question, heartbeat_interval=120, timeout=30): forward_logs, parent_sdk_version, save_diagnostics, + originator, ) = self._parse_question(question) except jsonschema.ValidationError: return - topic = Topic(name=self._pub_sub_id, project_name=self.backend.project_name) heartbeater = None + order = EventCounter() try: - self._send_delivery_acknowledgment(topic, question_uuid) + self._send_delivery_acknowledgment(question_uuid, originator, order) heartbeater = RepeatingTimer( interval=heartbeat_interval, function=self._send_heartbeat, - kwargs={"topic": topic, "question_uuid": question_uuid}, + kwargs={"question_uuid": question_uuid, "originator": originator, "order": order}, ) heartbeater.daemon = True heartbeater.start() if forward_logs: - analysis_log_handler = GooglePubSubHandler( - message_sender=self._send_message, - topic=topic, + analysis_log_handler = GoogleCloudPubSubHandler( + event_emitter=self._emit_event, question_uuid=question_uuid, + originator=originator, + recipient=originator, + order=order, ) else: analysis_log_handler = None @@ -242,23 +259,23 @@ def answer(self, question, heartbeat_interval=120, timeout=30): analysis_log_handler=analysis_log_handler, handle_monitor_message=functools.partial( self._send_monitor_message, - topic=topic, question_uuid=question_uuid, + originator=originator, + order=order, ), save_diagnostics=save_diagnostics, ) - result = {"kind": "result"} - - if analysis.output_values is not None: - result["output_values"] = analysis.output_values + result = make_minimal_dictionary(kind="result", output_values=analysis.output_values) if analysis.output_manifest is not None: result["output_manifest"] = analysis.output_manifest.to_primitive() - self._send_message( - message=result, - topic=topic, + self._emit_event( + event=result, + originator=originator, + recipient=originator, + order=order, attributes={"question_uuid": question_uuid, "sender_type": CHILD_SENDER_TYPE}, timeout=timeout, ) @@ -271,7 +288,7 @@ def answer(self, question, heartbeat_interval=120, timeout=30): heartbeater.cancel() warn_if_incompatible(child_sdk_version=self._local_sdk_version, parent_sdk_version=parent_sdk_version) - self.send_exception(topic, question_uuid, timeout=timeout) + self.send_exception(question_uuid, originator, order, timeout=timeout) raise error def ask( @@ -285,6 +302,7 @@ def ask( save_diagnostics="SAVE_DIAGNOSTICS_ON_CRASH", # This is repeated as a string here to avoid a circular import. question_uuid=None, push_endpoint=None, + asynchronous=False, timeout=86400, ): """Ask a child a question (i.e. send it input values for it to analyse and produce output values for) and return @@ -299,9 +317,10 @@ def ask( :param bool allow_local_files: if `True`, allow the input manifest to contain references to local files - this should only be set to `True` if the child will be able to access these local files :param str save_diagnostics: must be one of {"SAVE_DIAGNOSTICS_OFF", "SAVE_DIAGNOSTICS_ON_CRASH", "SAVE_DIAGNOSTICS_ON"}; if turned on, allow the input values and manifest (and its datasets) to be saved by the child either all the time or just if it fails while processing them :param str|None question_uuid: the UUID to use for the question if a specific one is needed; a UUID is generated if not - :param str|None push_endpoint: if answers to the question should be pushed to an endpoint, provide its URL here; if they should be pulled, leave this as `None` + :param str|None push_endpoint: if answers to the question should be pushed to an endpoint, provide its URL here (the returned subscription will be a push subscription); if not, leave this as `None` + :param bool asynchronous: if `True` and not using a push endpoint, don't create an answer subscription :param float|None timeout: time in seconds to keep retrying sending the question - :return (octue.cloud.pub_sub.subscription.Subscription, str): the answer subscription and question UUID + :return (octue.cloud.pub_sub.subscription.Subscription|None, str): the answer subscription (if the question is synchronous or a push endpoint was used) and question UUID """ service_namespace, service_name, service_revision_tag = split_service_id(service_id) @@ -328,54 +347,42 @@ def ask( "the new cloud locations." ) - pub_sub_service_id = convert_service_id_to_pub_sub_form(service_id) - topic = Topic(name=pub_sub_service_id, project_name=self.backend.project_name) - - if not topic.exists(timeout=timeout): - raise octue.exceptions.ServiceNotFound(f"Service with ID {service_id!r} cannot be found.") - question_uuid = question_uuid or str(uuid.uuid4()) - answer_subscription = Subscription( - name=".".join((topic.name, ANSWERS_NAMESPACE, question_uuid)), - topic=topic, - project_name=self.backend.project_name, - filter=f'attributes.question_uuid = "{question_uuid}" AND attributes.sender_type = "{CHILD_SENDER_TYPE}"', - push_endpoint=push_endpoint, - ) - answer_subscription.create(allow_existing=False) - - question = {"kind": "question"} - - if input_values is not None: - question["input_values"] = input_values - - if input_manifest is not None: - input_manifest.use_signed_urls_for_datasets() - question["input_manifest"] = input_manifest.to_primitive() - - if children is not None: - question["children"] = children - - self._send_message( - message=question, - topic=topic, - attributes={ - "question_uuid": question_uuid, - "sender_type": PARENT_SENDER_TYPE, - "forward_logs": subscribe_to_logs, - "save_diagnostics": save_diagnostics, - }, + if asynchronous and not push_endpoint: + answer_subscription = None + else: + pub_sub_id = convert_service_id_to_pub_sub_form(self.id) + + answer_subscription = Subscription( + name=".".join((OCTUE_SERVICES_PREFIX, pub_sub_id, ANSWERS_NAMESPACE, question_uuid)), + topic=self.services_topic, + filter=( + f'attributes.recipient = "{self.id}" ' + f'AND attributes.question_uuid = "{question_uuid}" ' + f'AND attributes.sender_type = "{CHILD_SENDER_TYPE}"' + ), + push_endpoint=push_endpoint, + ) + answer_subscription.create(allow_existing=False) + + self._send_question( + input_values=input_values, + input_manifest=input_manifest, + children=children, + forward_logs=subscribe_to_logs, + save_diagnostics=save_diagnostics, + question_uuid=question_uuid, + recipient=service_id, ) - logger.info("%r asked a question %r to service %r.", self, question_uuid, service_id) return answer_subscription, question_uuid def wait_for_answer( self, subscription, handle_monitor_message=None, - record_messages=True, + record_events=True, timeout=60, maximum_heartbeat_interval=300, ): @@ -384,7 +391,7 @@ def wait_for_answer( :param octue.cloud.pub_sub.subscription.Subscription subscription: the subscription for the question's answer :param callable|None handle_monitor_message: a function to handle monitor messages (e.g. send them to an endpoint for plotting or displaying) - this function should take a single JSON-compatible python primitive as an argument (note that this could be an array or object) - :param bool record_messages: if `True`, record messages received from the child in the `received_messages` attribute + :param bool record_events: if `True`, record messages received from the child in the `received_events` attribute :param float|None timeout: how long in seconds to wait for an answer before raising a `TimeoutError` :param float|int delivery_acknowledgement_timeout: how long in seconds to wait for a delivery acknowledgement before aborting :param float|int maximum_heartbeat_interval: the maximum amount of time (in seconds) allowed between child heartbeats before an error is raised @@ -392,23 +399,20 @@ def wait_for_answer( :return dict: dictionary containing the keys "output_values" and "output_manifest" """ if subscription.is_push_subscription: - raise octue.exceptions.PushSubscriptionCannotBePulled( + raise octue.exceptions.NotAPullSubscription( f"{subscription.path!r} is a push subscription so it cannot be waited on for an answer. Please check " f"its push endpoint at {subscription.push_endpoint!r}." ) - service_name = get_sruid_from_pub_sub_resource_name(subscription.name) - - self._message_handler = OrderedMessageHandler( + self._event_handler = GoogleCloudPubSubEventHandler( subscription=subscription, - receiving_service=self, + recipient=self, handle_monitor_message=handle_monitor_message, - service_name=service_name, - record_messages=record_messages, + record_events=record_events, ) try: - return self._message_handler.handle_messages( + return self._event_handler.handle_events( timeout=timeout, maximum_heartbeat_interval=maximum_heartbeat_interval, ) @@ -416,44 +420,54 @@ def wait_for_answer( finally: subscription.delete() - def send_exception(self, topic, question_uuid, timeout=30): + def send_exception(self, question_uuid, originator, order, timeout=30): """Serialise and send the exception being handled to the parent. - :param octue.cloud.pub_sub.topic.Topic topic: - :param str question_uuid: + :param str question_uuid: the UUID of the question this event relates to + :param str originator: the SRUID of the service that asked the question this event is related to + :param octue.cloud.events.counter.EventCounter order: an event counter keeping track of the order of emitted events :param float|None timeout: time in seconds to keep retrying sending of the exception :return None: """ exception = convert_exception_to_primitives() exception_message = f"Error in {self!r}: {exception['message']}" - self._send_message( + self._emit_event( { "kind": "exception", "exception_type": exception["type"], "exception_message": exception_message, "exception_traceback": exception["traceback"], }, - topic=topic, + originator=originator, + recipient=originator, + order=order, attributes={"question_uuid": question_uuid, "sender_type": CHILD_SENDER_TYPE}, timeout=timeout, ) - def _send_message(self, message, topic, attributes=None, timeout=30): - """Send a JSON-serialised message to the given topic with optional message attributes and increment the - `messages_published` attribute of the topic by one. This method is thread-safe. - - :param dict message: JSON-serialisable data to send as a message - :param octue.cloud.pub_sub.topic.Topic topic: the Pub/Sub topic to send the message to - :param dict|None attributes: key-value pairs to attach to the message - the values must be strings or bytes - :param int|float timeout: the timeout for sending the message in seconds - :return None: + def _emit_event(self, event, originator, recipient, order, attributes=None, timeout=30): + """Emit a JSON-serialised event as a Pub/Sub message to the services topic with optional message attributes, + incrementing the `order` argument by one. This method is thread-safe. + + :param dict event: JSON-serialisable data to emit as an event + :param str originator: the SRUID of the service that asked the question this event is related to + :param str recipient: the SRUID of the service the event is intended for + :param octue.cloud.events.counter.EventCounter order: an event counter keeping track of the order of emitted events + :param dict|None attributes: key-value pairs to attach to the event - the values must be strings or bytes + :param int|float timeout: the timeout for sending the event in seconds + :return google.cloud.pubsub_v1.publisher.futures.Future: """ attributes = attributes or {} - - with send_message_lock: - attributes["version"] = self._local_sdk_version - attributes["message_number"] = topic.messages_published + attributes["uuid"] = str(uuid.uuid4()) + attributes["originator"] = originator + attributes["sender"] = self.id + attributes["sender_sdk_version"] = self._local_sdk_version + attributes["recipient"] = recipient + + with emit_event_lock: + attributes["order"] = int(order) + attributes["datetime"] = datetime.datetime.utcnow().isoformat() converted_attributes = {} for key, value in attributes.items(): @@ -464,67 +478,125 @@ def _send_message(self, message, topic, attributes=None, timeout=30): converted_attributes[key] = value - self.publisher.publish( - topic=topic.path, - data=json.dumps(message, cls=OctueJSONEncoder).encode(), + future = self.publisher.publish( + topic=self.services_topic.path, + data=json.dumps(event, cls=OctueJSONEncoder).encode(), retry=retry.Retry(deadline=timeout), **converted_attributes, ) - topic.messages_published += 1 + order += 1 + + return future + + def _send_question( + self, + input_values, + input_manifest, + children, + forward_logs, + save_diagnostics, + question_uuid, + recipient, + timeout=30, + ): + """Send a question to a child service. + + :param any|None input_values: any input values for the question + :param octue.resources.manifest.Manifest|None input_manifest: an input manifest of any datasets needed for the question + :param list(dict)|None children: a list of children for the child to use instead of its default children (if it uses children). These should be in the same format as in an app's app configuration file and have the same keys. + :param bool forward_logs: whether to request the child to forward its logs + :param str save_diagnostics: must be one of {"SAVE_DIAGNOSTICS_OFF", "SAVE_DIAGNOSTICS_ON_CRASH", "SAVE_DIAGNOSTICS_ON"}; if turned on, allow the input values and manifest (and its datasets) to be saved by the child either all the time or just if it fails while processing them + :param str question_uuid: the UUID of the question being sent + :param str recipient: the SRUID of the child the question is intended for + :param float timeout: time in seconds after which to give up sending + :return None: + """ + question = make_minimal_dictionary(kind="question", input_values=input_values, children=children) + + if input_manifest is not None: + input_manifest.use_signed_urls_for_datasets() + question["input_manifest"] = input_manifest.to_primitive() + + future = self._emit_event( + event=question, + timeout=timeout, + originator=self.id, + recipient=recipient, + order=EventCounter(), + attributes={ + "question_uuid": question_uuid, + "forward_logs": forward_logs, + "save_diagnostics": save_diagnostics, + "sender_type": PARENT_SENDER_TYPE, + }, + ) + + # Await successful publishing of the question. + future.result() + logger.info("%r asked a question %r to service %r.", self, question_uuid, recipient) - def _send_delivery_acknowledgment(self, topic, question_uuid, timeout=30): + def _send_delivery_acknowledgment(self, question_uuid, originator, order, timeout=30): """Send an acknowledgement of question receipt to the parent. - :param octue.cloud.pub_sub.topic.Topic topic: topic to send the acknowledgement to - :param str question_uuid: + :param str question_uuid: the UUID of the question this event relates to + :param str originator: the SRUID of the service that asked the question this event is related to + :param octue.cloud.events.counter.EventCounter order: an event counter keeping track of the order of emitted events :param float timeout: time in seconds after which to give up sending :return None: """ - self._send_message( + self._emit_event( { "kind": "delivery_acknowledgement", "datetime": datetime.datetime.utcnow().isoformat(), }, - topic=topic, timeout=timeout, + originator=originator, + recipient=originator, + order=order, attributes={"question_uuid": question_uuid, "sender_type": CHILD_SENDER_TYPE}, ) - logger.info("%r acknowledged receipt of question.", self) + logger.info("%r acknowledged receipt of question %r.", self, question_uuid) - def _send_heartbeat(self, topic, question_uuid, timeout=30): + def _send_heartbeat(self, question_uuid, originator, order, timeout=30): """Send a heartbeat to the parent, indicating that the service is alive. - :param octue.cloud.pub_sub.topic.Topic topic: topic to send the heartbeat to - :param str question_uuid: + :param str question_uuid: the UUID of the question this event relates to + :param str originator: the SRUID of the service that asked the question this event is related to + :param octue.cloud.events.counter.EventCounter order: an event counter keeping track of the order of emitted events :param float timeout: time in seconds after which to give up sending :return None: """ - self._send_message( + self._emit_event( { "kind": "heartbeat", "datetime": datetime.datetime.utcnow().isoformat(), }, - topic=topic, + originator=originator, + recipient=originator, + order=order, timeout=timeout, attributes={"question_uuid": question_uuid, "sender_type": CHILD_SENDER_TYPE}, ) logger.debug("Heartbeat sent by %r.", self) - def _send_monitor_message(self, data, topic, question_uuid, timeout=30): + def _send_monitor_message(self, data, question_uuid, originator, order, timeout=30): """Send a monitor message to the parent. :param any data: the data to send as a monitor message - :param octue.cloud.pub_sub.topic.Topic topic: the topic to send the message to - :param str question_uuid: + :param str question_uuid: the UUID of the question this event relates to + :param str originator: the SRUID of the service that asked the question this event is related to + :param octue.cloud.events.counter.EventCounter order: an event counter keeping track of the order of emitted events :param float timeout: time in seconds to retry sending the message :return None: """ - self._send_message( + self._emit_event( {"kind": "monitor_message", "data": data}, - topic=topic, + originator=originator, + recipient=originator, + order=order, timeout=timeout, attributes={"question_uuid": question_uuid, "sender_type": CHILD_SENDER_TYPE}, ) @@ -535,31 +607,33 @@ def _parse_question(self, question): """Parse a question in the Google Cloud Run or Google Pub/Sub format. :param dict|google.cloud.pubsub_v1.subscriber.message.Message question: the question to parse in Google Cloud Run or Google Pub/Sub format - :return (dict, str, bool, str, str): the question's event and its attributes (question UUID, whether to forward logs, the Octue SDK version of the parent, and whether to save diagnostics) + :return (dict, str, bool, str, str, str): the question's event and its attributes (question UUID, whether to forward logs, the Octue SDK version of the parent, whether to save diagnostics, and the SRUID of the service revision that asked the question) """ logger.info("%r received a question.", self) - # Acknowledge it if it's directly from Pub/Sub + # Acknowledge the question if it's directly from Pub/Sub. if hasattr(question, "ack"): question.ack() - event, attributes = extract_event_and_attributes_from_pub_sub(question) + event, attributes = extract_event_and_attributes_from_pub_sub_message(question) event_for_validation = copy.deepcopy(event) raise_if_event_is_invalid( event=event_for_validation, attributes=attributes, - receiving_service=self, - parent_sdk_version=attributes.get("version"), + recipient=self, + # Don't assume the presence of specific attributes before validation. + parent_sdk_version=attributes.get("sender_sdk_version"), child_sdk_version=importlib.metadata.version("octue"), ) - logger.info("%r parsed the question successfully.", self) + logger.info("%r parsed question %r successfully.", self, attributes["question_uuid"]) return ( event, attributes["question_uuid"], attributes["forward_logs"], - attributes["version"], + attributes["sender_sdk_version"], attributes["save_diagnostics"], + attributes["originator"], ) diff --git a/octue/cloud/pub_sub/subscription.py b/octue/cloud/pub_sub/subscription.py index 560f7da42..53f06f6fd 100644 --- a/octue/cloud/pub_sub/subscription.py +++ b/octue/cloud/pub_sub/subscription.py @@ -12,8 +12,6 @@ UpdateSubscriptionRequest, ) -from octue.cloud.service_id import OCTUE_SERVICES_NAMESPACE - logger = logging.getLogger(__name__) @@ -27,14 +25,14 @@ class Subscription: :param str name: the name of the subscription excluding "projects//subscriptions/" :param octue.cloud.pub_sub.topic.Topic topic: the topic the subscription is attached to - :param str project_name: the name of the Google Cloud project that the subscription belongs to + :param str|None project_name: the name of the Google Cloud project that the subscription belongs to; if `None`, the project name of the topic is used :param str|None filter: if provided, only receive messages matching the filter (see here for filter syntax: https://cloud.google.com/pubsub/docs/subscription-message-filter#filtering_syntax) :param int ack_deadline: the time in seconds after which, if the subscriber hasn't acknowledged a message, to retry sending it to the subscription :param int message_retention_duration: unacknowledged message retention time in seconds :param int|float|None expiration_time: number of seconds of inactivity after which the subscription is deleted (infinite time if `None`) :param float minimum_retry_backoff: minimum number of seconds after the acknowledgement deadline has passed to exponentially retry delivering a message to the subscription :param float maximum_retry_backoff: maximum number of seconds after the acknowledgement deadline has passed to exponentially retry delivering a message to the subscription - :param str|None push_endpoint: if this is a push subscription, this is the URL to which messages should be pushed; leave as `None` if this is a pull subscription + :param str|None push_endpoint: if this is a push subscription, this is the URL to which messages should be pushed; leave as `None` if it's not a push subscription :return None: """ @@ -42,7 +40,7 @@ def __init__( self, name, topic, - project_name, + project_name=None, filter=None, ack_deadline=600, message_retention_duration=600, @@ -51,14 +49,10 @@ def __init__( maximum_retry_backoff=600, push_endpoint=None, ): - if not name.startswith(OCTUE_SERVICES_NAMESPACE): - self.name = f"{OCTUE_SERVICES_NAMESPACE}.{name}" - else: - self.name = name - + self.name = name self.topic = topic self.filter = filter - self.path = self.generate_subscription_path(project_name, self.name) + self.path = self.generate_subscription_path(project_name or self.topic.project_name, self.name) self.ack_deadline = ack_deadline self.message_retention_duration = Duration(seconds=message_retention_duration) @@ -184,9 +178,9 @@ def _create_proto_message_subscription(self): :return google.pubsub_v1.types.pubsub.Subscription: """ if self.push_endpoint: - push_config = {"push_config": PushConfig(mapping=None, push_endpoint=self.push_endpoint)} # noqa + options = {"push_config": PushConfig(mapping=None, push_endpoint=self.push_endpoint)} # noqa else: - push_config = {} + options = {} return _Subscription( mapping=None, @@ -197,7 +191,7 @@ def _create_proto_message_subscription(self): message_retention_duration=self.message_retention_duration, # noqa expiration_policy=self.expiration_policy, # noqa retry_policy=self.retry_policy, # noqa - **push_config, + **options, ) def _log_creation(self): diff --git a/octue/cloud/pub_sub/topic.py b/octue/cloud/pub_sub/topic.py index 4b2be732b..42d33fafe 100644 --- a/octue/cloud/pub_sub/topic.py +++ b/octue/cloud/pub_sub/topic.py @@ -6,8 +6,6 @@ from google.cloud.pubsub_v1 import PublisherClient from google.pubsub_v1.types.pubsub import Topic as Topic_ -from octue.cloud.service_id import OCTUE_SERVICES_NAMESPACE - logger = logging.getLogger(__name__) @@ -23,13 +21,9 @@ class Topic: """ def __init__(self, name, project_name): - if not name.startswith(OCTUE_SERVICES_NAMESPACE): - self.name = f"{OCTUE_SERVICES_NAMESPACE}.{name}" - else: - self.name = name - - self.path = self.generate_topic_path(project_name, self.name) - self.messages_published = 0 + self.name = name + self.project_name = project_name + self.path = self.generate_topic_path(self.project_name, self.name) self._publisher = PublisherClient() self._created = False @@ -47,7 +41,7 @@ def __repr__(self): :return str: """ - return f"<{type(self).__name__}({self.name})>" + return f"<{type(self).__name__}(name={self.name!r})>" def create(self, allow_existing=False): """Create a Google Pub/Sub topic that can be published to. diff --git a/octue/cloud/service_id.py b/octue/cloud/service_id.py index fa2e021a8..dd041a38a 100644 --- a/octue/cloud/service_id.py +++ b/octue/cloud/service_id.py @@ -10,9 +10,6 @@ logger = logging.getLogger(__name__) - -OCTUE_SERVICES_NAMESPACE = "octue.services" - SERVICE_NAMESPACE_AND_NAME_PATTERN = r"([a-z0-9])+(-([a-z0-9])+)*" COMPILED_SERVICE_NAMESPACE_AND_NAME_PATTERN = re.compile(SERVICE_NAMESPACE_AND_NAME_PATTERN) @@ -216,7 +213,7 @@ def get_sruid_from_pub_sub_resource_name(name): :param str name: the name of the topic or subscription :return str: the SRUID of the service revision the topic or subscription is related to """ - _, _, namespace, name, revision_tag, *_ = name.split(".") + namespace, name, revision_tag, *_ = name.split(".") return f"{namespace}/{name}:{revision_tag.replace('-', '.')}" diff --git a/octue/configuration.py b/octue/configuration.py index 1a1f2f7d1..c5f487013 100644 --- a/octue/configuration.py +++ b/octue/configuration.py @@ -36,6 +36,8 @@ def __init__( ): self.name = name self.namespace = namespace + self.diagnostics_cloud_path = diagnostics_cloud_path + self.service_registries = service_registries if directory: directory = os.path.abspath(directory) @@ -61,9 +63,6 @@ def __init__( else: self.app_configuration_path = None - self.diagnostics_cloud_path = diagnostics_cloud_path - self.service_registries = service_registries - if kwargs: logger.warning(f"The following keyword arguments were not used by {type(self).__name__}: {kwargs!r}.") diff --git a/octue/exceptions.py b/octue/exceptions.py index 8b31a5478..93ee75479 100644 --- a/octue/exceptions.py +++ b/octue/exceptions.py @@ -114,8 +114,8 @@ class CloudStorageBucketNotFound(OctueSDKException): """Raise if attempting to access a cloud storage bucket that cannot be found.""" -class PushSubscriptionCannotBePulled(OctueSDKException): - """Raise if attempting to pull a push subscription.""" +class NotAPullSubscription(OctueSDKException): + """Raise if attempting to pull a subscription that's not a pull subscription.""" class ReadOnlyResource(OctueSDKException): diff --git a/octue/metadata/recorded_questions.jsonl b/octue/metadata/recorded_questions.jsonl index ee9e31f53..c658619e1 100644 --- a/octue/metadata/recorded_questions.jsonl +++ b/octue/metadata/recorded_questions.jsonl @@ -1,15 +1,3 @@ -{"parent_sdk_version": "0.30.0", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": \"{\\n \\\"datasets\\\": {\\n \\\"my_dataset\\\": \\\"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmpw94cjhfq\\\"\\n },\\n \\\"id\\\": \\\"cf38d606-c9d2-4f54-a806-6337b1fa30fb\\\",\\n \\\"name\\\": null\\n}\"}", "attributes": {"question_uuid": "8157304e-e056-4ec9-80ef-dce540c27191", "forward_logs": "1", "octue_sdk_version": "0.30.0"}}} -{"parent_sdk_version": "0.31.0", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": \"{\\n \\\"datasets\\\": {\\n \\\"my_dataset\\\": \\\"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmpzrl_zrel\\\"\\n },\\n \\\"id\\\": \\\"b2d0a0cf-cdc6-48d8-92a7-d7cbee14807f\\\",\\n \\\"name\\\": null\\n}\"}", "attributes": {"question_uuid": "130b92bb-4486-4785-92ab-96f641367b69", "forward_logs": "1", "octue_sdk_version": "0.31.0", "allow_save_diagnostics_data_on_crash": "1"}}} -{"parent_sdk_version": "0.32.0", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": \"{\\n \\\"datasets\\\": {\\n \\\"my_dataset\\\": \\\"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmpg9dgmfjy\\\"\\n },\\n \\\"id\\\": \\\"7f33f9f5-79e6-46af-b450-50c1b7ad19d5\\\",\\n \\\"name\\\": null\\n}\"}", "attributes": {"question_uuid": "9e982e65-5937-4e89-9c69-d86640805ce4", "forward_logs": "1", "octue_sdk_version": "0.32.0", "allow_save_diagnostics_data_on_crash": "1"}}} -{"parent_sdk_version": "0.33.0", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": \"{\\n \\\"datasets\\\": {\\n \\\"my_dataset\\\": \\\"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmpeofjz_r9\\\"\\n },\\n \\\"id\\\": \\\"8b860309-4c9f-43a0-bf3a-621a59ad02cd\\\",\\n \\\"name\\\": null\\n}\"}", "attributes": {"question_uuid": "4414295f-fd07-4447-9170-fae08e525614", "forward_logs": "1", "octue_sdk_version": "0.33.0", "allow_save_diagnostics_data_on_crash": "1"}}} -{"parent_sdk_version": "0.34.0", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": \"{\\n \\\"datasets\\\": {\\n \\\"my_dataset\\\": \\\"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmp8enl0nnh\\\"\\n },\\n \\\"id\\\": \\\"0b602f70-741c-4ec7-bde4-d6e18ba5213f\\\",\\n \\\"name\\\": null\\n}\"}", "attributes": {"question_uuid": "8b0d7b63-ee2a-4d3b-8cfd-ab98fc389c78", "forward_logs": "1", "allow_save_diagnostics_data_on_crash": "1", "octue_sdk_version": "0.34.0"}}} -{"parent_sdk_version": "0.34.1", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": \"{\\n \\\"datasets\\\": {\\n \\\"my_dataset\\\": \\\"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmp1y2hp_71\\\"\\n },\\n \\\"id\\\": \\\"defb3d77-1d60-48e3-9270-4364badccb3d\\\",\\n \\\"name\\\": null\\n}\"}", "attributes": {"question_uuid": "15cba19c-401a-4321-9f73-67c78f28016e", "forward_logs": "1", "allow_save_diagnostics_data_on_crash": "1", "octue_sdk_version": "0.34.1"}}} -{"parent_sdk_version": "0.35.0", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": \"{\\n \\\"datasets\\\": {\\n \\\"my_dataset\\\": \\\"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmpsx4bxiyq\\\"\\n },\\n \\\"id\\\": \\\"56538ac2-7e4b-4f8b-8d03-bfba60881cea\\\",\\n \\\"name\\\": null\\n}\"}", "attributes": {"question_uuid": "f8f30ded-714f-4719-9622-61e850ceb872", "forward_logs": "1", "allow_save_diagnostics_data_on_crash": "1", "octue_sdk_version": "0.35.0"}}} -{"parent_sdk_version": "0.36.0", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": \"{\\n \\\"datasets\\\": {\\n \\\"my_dataset\\\": \\\"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmpj6b99swv\\\"\\n },\\n \\\"id\\\": \\\"1578e98c-a449-42e1-99c3-9f16eb64e04e\\\",\\n \\\"name\\\": null\\n}\"}", "attributes": {"question_uuid": "8fc1eecd-e6f4-4985-8a2f-9873f1401eed", "forward_logs": "1", "allow_save_diagnostics_data_on_crash": "1", "octue_sdk_version": "0.36.0"}}} -{"parent_sdk_version": "0.37.0", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": \"{\\n \\\"datasets\\\": {\\n \\\"my_dataset\\\": \\\"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmp4394fsdk\\\"\\n },\\n \\\"id\\\": \\\"98fe6a81-e6d0-4e1b-bb70-2d7490024e3e\\\",\\n \\\"name\\\": null\\n}\"}", "attributes": {"question_uuid": "07d706d9-5df3-44a9-a133-fc56c235559a", "forward_logs": "1", "allow_save_diagnostics_data_on_crash": "1", "octue_sdk_version": "0.37.0"}}} -{"parent_sdk_version": "0.38.0", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": \"{\\n \\\"datasets\\\": {\\n \\\"my_dataset\\\": \\\"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmpc_nmpiz2\\\"\\n },\\n \\\"id\\\": \\\"739b1ae5-99f8-4bd0-b344-38f35d74b8a8\\\",\\n \\\"name\\\": null\\n}\"}", "attributes": {"question_uuid": "7bb5e29f-c8de-4fb4-923b-b5c854a24f49", "forward_logs": "1", "allow_save_diagnostics_data_on_crash": "1", "octue_sdk_version": "0.38.0"}}} -{"parent_sdk_version": "0.38.1", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": \"{\\n \\\"datasets\\\": {\\n \\\"my_dataset\\\": \\\"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmpt1uu7685\\\"\\n },\\n \\\"id\\\": \\\"8b3c39f9-5232-4898-998d-934defdc5626\\\",\\n \\\"name\\\": null\\n}\"}", "attributes": {"question_uuid": "a96e6ec5-3798-4160-b45f-6f248834f740", "forward_logs": "1", "allow_save_diagnostics_data_on_crash": "1", "octue_sdk_version": "0.38.1"}}} -{"parent_sdk_version": "0.39.0", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": \"{\\n \\\"datasets\\\": {\\n \\\"my_dataset\\\": \\\"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmp664lr951\\\"\\n },\\n \\\"id\\\": \\\"57a78545-f40f-4971-bff9-b138bf1d6292\\\",\\n \\\"name\\\": null\\n}\", \"children\": null}", "attributes": {"question_uuid": "7a8eed0b-bd2d-4eb7-93b3-5eb8244c812d", "forward_logs": "1", "allow_save_diagnostics_data_on_crash": "1", "octue_sdk_version": "0.39.0"}}} {"parent_sdk_version": "0.40.0", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": \"{\\n \\\"datasets\\\": {\\n \\\"my_dataset\\\": \\\"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmppnbc6rcc\\\"\\n },\\n \\\"id\\\": \\\"31440887-beef-41a3-935f-08adbe8dbbdc\\\",\\n \\\"name\\\": null\\n}\", \"children\": null}", "attributes": {"question_uuid": "f66edd6e-517c-4c26-9d60-16cb508e5fe1", "forward_logs": "1", "allow_save_diagnostics_data_on_crash": "1", "octue_sdk_version": "0.40.0"}}} {"parent_sdk_version": "0.40.1", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": \"{\\n \\\"datasets\\\": {\\n \\\"my_dataset\\\": \\\"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmppnbc6rcc\\\"\\n },\\n \\\"id\\\": \\\"31440887-beef-41a3-935f-08adbe8dbbdc\\\",\\n \\\"name\\\": null\\n}\", \"children\": null}", "attributes": {"question_uuid": "f66edd6e-517c-4c26-9d60-16cb508e5fe1", "forward_logs": "1", "allow_save_diagnostics_data_on_crash": "1", "octue_sdk_version": "0.40.1"}}} {"parent_sdk_version": "0.40.2", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": \"{\\n \\\"datasets\\\": {\\n \\\"my_dataset\\\": \\\"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmprj5sl5al\\\"\\n },\\n \\\"id\\\": \\\"6ce744fb-fce6-4346-949a-109675ca7a5a\\\",\\n \\\"name\\\": null\\n}\", \"children\": null}", "attributes": {"question_uuid": "42d9a880-8672-4be6-a3ae-b8518a9e58db", "forward_logs": "1", "allow_save_diagnostics_data_on_crash": "1", "octue_sdk_version": "0.40.2"}}} @@ -43,3 +31,4 @@ {"parent_sdk_version": "0.51.0", "question": {"data": "{\"kind\": \"question\", \"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": {\"id\": \"3bf3b178-3cf1-49aa-a1d9-89cd8ec05b1a\", \"name\": null, \"datasets\": {\"my_dataset\": \"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmppu85nw5c\"}}}", "attributes": {"question_uuid": "6f7f6e1c-7632-4b4d-b91d-6f58dcb43c40", "sender_type": "PARENT", "forward_logs": "1", "save_diagnostics": "SAVE_DIAGNOSTICS_ON_CRASH", "version": "0.51.0", "message_number": "0"}}} {"parent_sdk_version": "0.52.0", "question": {"data": "{\"kind\": \"question\", \"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": {\"id\": \"6be875b3-33d8-4b00-b7ea-553f8f69bce5\", \"name\": null, \"datasets\": {\"my_dataset\": \"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmphypwu9uh\"}}}", "attributes": {"question_uuid": "cd0a78be-fda2-4730-bdba-ba6b04e4787f", "sender_type": "PARENT", "forward_logs": "1", "save_diagnostics": "SAVE_DIAGNOSTICS_ON_CRASH", "version": "0.52.0", "message_number": "0"}}} {"parent_sdk_version": "0.52.1", "question": {"data": "{\"kind\": \"question\", \"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": {\"id\": \"6be875b3-33d8-4b00-b7ea-553f8f69bce5\", \"name\": null, \"datasets\": {\"my_dataset\": \"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmphypwu9uh\"}}}", "attributes": {"question_uuid": "cd0a78be-fda2-4730-bdba-ba6b04e4787f", "sender_type": "PARENT", "forward_logs": "1", "save_diagnostics": "SAVE_DIAGNOSTICS_ON_CRASH", "version": "0.52.1", "message_number": "0"}}} +{"parent_sdk_version": "0.53.0", "question": {"data": "{\"kind\": \"question\", \"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": {\"id\": \"6be875b3-33d8-4b00-b7ea-553f8f69bce5\", \"name\": null, \"datasets\": {\"my_dataset\": \"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmphypwu9uh\"}}}", "attributes": {"datetime": "2024-04-11T10:46:48.236064", "uuid": "a9de11b1-e88f-43fa-b3a4-40a590c3443f", "question_uuid": "cd0a78be-fda2-4730-bdba-ba6b04e4787f", "forward_logs": "1", "save_diagnostics": "SAVE_DIAGNOSTICS_ON_CRASH", "originator": "octue/test-service:0.1.0", "sender": "octue/test-service:0.1.0", "sender_type": "PARENT", "sender_sdk_version": "0.53.0", "recipient": "octue/another-service:1.0.12", "order": "0"}}} diff --git a/octue/metadata/version_compatibilities.json b/octue/metadata/version_compatibilities.json index f18c95549..712c71f5f 100644 --- a/octue/metadata/version_compatibilities.json +++ b/octue/metadata/version_compatibilities.json @@ -1,583 +1,7 @@ { - "0.35.0": { - "0.40.1": true, - "0.38.1": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, - "0.36.0": true, - "0.37.0": true, - "0.38.0": true, - "0.39.0": true, - "0.40.0": true, - "0.40.2": true, - "0.41.0": true, - "0.41.1": true, - "0.42.0": true, - "0.42.1": true, - "0.43.0": true, - "0.43.1": true, - "0.43.2": true, - "0.43.3": true, - "0.43.4": true, - "0.43.5": true, - "0.43.6": true, - "0.43.7": true, - "0.44.0": true, - "0.45.0": true, - "0.46.0": true, - "0.46.1": true, - "0.46.2": true, - "0.46.3": true, - "0.47.0": true, - "0.47.1": true, - "0.47.2": true, - "0.48.0": true, - "0.49.0": true, - "0.49.1": true, - "0.49.2": true, - "0.50.0": true, - "0.50.1": true, - "0.51.0": false, - "0.52.0": false, - "0.52.1": false - }, - "0.34.1": { - "0.40.1": true, - "0.38.1": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, - "0.36.0": true, - "0.37.0": true, - "0.38.0": true, - "0.39.0": true, - "0.40.0": true, - "0.40.2": true, - "0.41.0": true, - "0.41.1": true, - "0.42.0": true, - "0.42.1": true, - "0.43.0": true, - "0.43.1": true, - "0.43.2": true, - "0.43.3": true, - "0.43.4": true, - "0.43.5": true, - "0.43.6": true, - "0.43.7": true, - "0.44.0": true, - "0.45.0": true, - "0.46.0": true, - "0.46.1": true, - "0.46.2": true, - "0.46.3": true, - "0.47.0": true, - "0.47.1": true, - "0.47.2": true, - "0.48.0": true, - "0.49.0": true, - "0.49.1": true, - "0.49.2": true, - "0.50.0": true, - "0.50.1": true, - "0.51.0": false, - "0.52.0": false, - "0.52.1": false - }, - "0.34.0": { - "0.40.1": true, - "0.38.1": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, - "0.36.0": true, - "0.37.0": true, - "0.38.0": true, - "0.39.0": true, - "0.40.0": true, - "0.40.2": true, - "0.41.0": true, - "0.41.1": true, - "0.42.0": true, - "0.42.1": true, - "0.43.0": true, - "0.43.1": true, - "0.43.2": true, - "0.43.3": true, - "0.43.4": true, - "0.43.5": true, - "0.43.6": true, - "0.43.7": true, - "0.44.0": true, - "0.45.0": true, - "0.46.0": true, - "0.46.1": true, - "0.46.2": true, - "0.46.3": true, - "0.47.0": true, - "0.47.1": true, - "0.47.2": true, - "0.48.0": true, - "0.49.0": true, - "0.49.1": true, - "0.49.2": true, - "0.50.0": true, - "0.50.1": true, - "0.51.0": false, - "0.52.0": false, - "0.52.1": false - }, - "0.33.0": { - "0.40.1": true, - "0.38.1": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, - "0.36.0": true, - "0.37.0": true, - "0.38.0": true, - "0.39.0": true, - "0.40.0": true, - "0.40.2": true, - "0.41.0": true, - "0.41.1": true, - "0.42.0": true, - "0.42.1": true, - "0.43.0": true, - "0.43.1": true, - "0.43.2": true, - "0.43.3": true, - "0.43.4": true, - "0.43.5": true, - "0.43.6": true, - "0.43.7": true, - "0.44.0": true, - "0.45.0": true, - "0.46.0": true, - "0.46.1": true, - "0.46.2": true, - "0.46.3": true, - "0.47.0": true, - "0.47.1": true, - "0.47.2": true, - "0.48.0": true, - "0.49.0": true, - "0.49.1": true, - "0.49.2": true, - "0.50.0": true, - "0.50.1": true, - "0.51.0": false, - "0.52.0": false, - "0.52.1": false - }, - "0.32.0": { - "0.40.1": true, - "0.38.1": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, - "0.36.0": true, - "0.37.0": true, - "0.38.0": true, - "0.39.0": true, - "0.40.0": true, - "0.40.2": true, - "0.41.0": true, - "0.41.1": true, - "0.42.0": true, - "0.42.1": true, - "0.43.0": true, - "0.43.1": true, - "0.43.2": true, - "0.43.3": true, - "0.43.4": true, - "0.43.5": true, - "0.43.6": true, - "0.43.7": true, - "0.44.0": true, - "0.45.0": true, - "0.46.0": true, - "0.46.1": true, - "0.46.2": true, - "0.46.3": true, - "0.47.0": true, - "0.47.1": true, - "0.47.2": true, - "0.48.0": true, - "0.49.0": true, - "0.49.1": true, - "0.49.2": true, - "0.50.0": true, - "0.50.1": true, - "0.51.0": false, - "0.52.0": false, - "0.52.1": false - }, - "0.31.0": { - "0.40.1": true, - "0.38.1": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, - "0.36.0": true, - "0.37.0": true, - "0.38.0": true, - "0.39.0": true, - "0.40.0": true, - "0.40.2": true, - "0.41.0": true, - "0.41.1": true, - "0.42.0": true, - "0.42.1": true, - "0.43.0": true, - "0.43.1": true, - "0.43.2": true, - "0.43.3": true, - "0.43.4": true, - "0.43.5": true, - "0.43.6": true, - "0.43.7": true, - "0.44.0": true, - "0.45.0": true, - "0.46.0": true, - "0.46.1": true, - "0.46.2": true, - "0.46.3": true, - "0.47.0": true, - "0.47.1": true, - "0.47.2": true, - "0.48.0": true, - "0.49.0": true, - "0.49.1": true, - "0.49.2": true, - "0.50.0": true, - "0.50.1": true, - "0.51.0": false, - "0.52.0": false, - "0.52.1": false - }, - "0.30.0": { - "0.40.1": true, - "0.38.1": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, - "0.36.0": true, - "0.37.0": true, - "0.38.0": true, - "0.39.0": true, - "0.40.0": true, - "0.40.2": true, - "0.41.0": true, - "0.41.1": true, - "0.42.0": true, - "0.42.1": true, - "0.43.0": true, - "0.43.1": true, - "0.43.2": true, - "0.43.3": true, - "0.43.4": true, - "0.43.5": true, - "0.43.6": true, - "0.43.7": true, - "0.44.0": true, - "0.45.0": true, - "0.46.0": true, - "0.46.1": true, - "0.46.2": true, - "0.46.3": true, - "0.47.0": true, - "0.47.1": true, - "0.47.2": true, - "0.48.0": true, - "0.49.0": true, - "0.49.1": true, - "0.49.2": true, - "0.50.0": true, - "0.50.1": true, - "0.51.0": false, - "0.52.0": false, - "0.52.1": false - }, - "0.36.0": { - "0.40.1": true, - "0.38.1": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, - "0.37.0": true, - "0.38.0": true, - "0.39.0": true, - "0.40.0": true, - "0.40.2": true, - "0.41.0": true, - "0.41.1": true, - "0.42.0": true, - "0.42.1": true, - "0.43.0": true, - "0.43.1": true, - "0.43.2": true, - "0.43.3": true, - "0.43.4": true, - "0.43.5": true, - "0.43.6": true, - "0.43.7": true, - "0.44.0": true, - "0.45.0": true, - "0.46.0": true, - "0.46.1": true, - "0.46.2": true, - "0.46.3": true, - "0.47.0": true, - "0.47.1": true, - "0.47.2": true, - "0.48.0": true, - "0.49.0": true, - "0.49.1": true, - "0.49.2": true, - "0.50.0": true, - "0.50.1": true, - "0.51.0": false, - "0.52.0": false, - "0.52.1": false - }, - "0.37.0": { - "0.40.1": true, - "0.38.1": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, - "0.38.0": true, - "0.39.0": true, - "0.40.0": true, - "0.40.2": true, - "0.41.0": true, - "0.41.1": true, - "0.42.0": true, - "0.42.1": true, - "0.43.0": true, - "0.43.1": true, - "0.43.2": true, - "0.43.3": true, - "0.43.4": true, - "0.43.5": true, - "0.43.6": true, - "0.43.7": true, - "0.44.0": true, - "0.45.0": true, - "0.46.0": true, - "0.46.1": true, - "0.46.2": true, - "0.46.3": true, - "0.47.0": true, - "0.47.1": true, - "0.47.2": true, - "0.48.0": true, - "0.49.0": true, - "0.49.1": true, - "0.49.2": true, - "0.50.0": true, - "0.50.1": true, - "0.51.0": false, - "0.52.0": false, - "0.52.1": false - }, - "0.38.0": { - "0.40.1": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, - "0.39.0": true, - "0.40.0": true, - "0.40.2": true, - "0.41.0": true, - "0.41.1": true, - "0.42.0": true, - "0.42.1": true, - "0.43.0": true, - "0.43.1": true, - "0.43.2": true, - "0.43.3": true, - "0.43.4": true, - "0.43.5": true, - "0.43.6": true, - "0.43.7": true, - "0.44.0": true, - "0.45.0": true, - "0.46.0": true, - "0.46.1": true, - "0.46.2": true, - "0.46.3": true, - "0.47.0": true, - "0.47.1": true, - "0.47.2": true, - "0.48.0": true, - "0.49.0": true, - "0.49.1": true, - "0.49.2": true, - "0.50.0": true, - "0.50.1": true, - "0.51.0": false, - "0.52.0": false, - "0.52.1": false - }, - "0.38.1": { - "0.40.1": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, - "0.39.0": true, - "0.40.0": true, - "0.40.2": true, - "0.41.0": true, - "0.41.1": true, - "0.42.0": true, - "0.42.1": true, - "0.43.0": true, - "0.43.1": true, - "0.43.2": true, - "0.43.3": true, - "0.43.4": true, - "0.43.5": true, - "0.43.6": true, - "0.43.7": true, - "0.44.0": true, - "0.45.0": true, - "0.46.0": true, - "0.46.1": true, - "0.46.2": true, - "0.46.3": true, - "0.47.0": true, - "0.47.1": true, - "0.47.2": true, - "0.48.0": true, - "0.49.0": true, - "0.49.1": true, - "0.49.2": true, - "0.50.0": true, - "0.50.1": true, - "0.51.0": false, - "0.52.0": false, - "0.52.1": false - }, - "0.39.0": { - "0.40.1": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, - "0.40.0": true, - "0.40.2": true, - "0.41.0": true, - "0.41.1": true, - "0.42.0": true, - "0.42.1": true, - "0.43.0": true, - "0.43.1": true, - "0.43.2": true, - "0.43.3": true, - "0.43.4": true, - "0.43.5": true, - "0.43.6": true, - "0.43.7": true, - "0.44.0": true, - "0.45.0": true, - "0.46.0": true, - "0.46.1": true, - "0.46.2": true, - "0.46.3": true, - "0.47.0": true, - "0.47.1": true, - "0.47.2": true, - "0.48.0": true, - "0.49.0": true, - "0.49.1": true, - "0.49.2": true, - "0.50.0": true, - "0.50.1": true, - "0.51.0": false, - "0.52.0": false, - "0.52.1": false - }, "0.40.0": { "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.40.2": true, "0.41.0": true, "0.41.1": true, @@ -608,23 +32,12 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.40.1": { "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.40.2": true, "0.41.0": true, "0.41.1": true, @@ -655,25 +68,14 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.40.2": { "0.41.0": true, "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.41.1": true, "0.42.0": true, "0.42.1": true, @@ -702,25 +104,14 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.41.0": { "0.41.0": true, "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.41.1": true, "0.42.0": true, "0.42.1": true, @@ -749,7 +140,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.41.1": { "0.41.1": true, @@ -757,18 +149,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.42.0": true, "0.42.1": true, "0.43.0": true, @@ -796,7 +176,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.42.0": { "0.42.0": true, @@ -805,18 +186,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.42.1": true, "0.43.0": true, "0.43.1": true, @@ -843,7 +212,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.42.1": { "0.43.2": true, @@ -856,18 +226,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.43.3": true, "0.43.4": true, "0.43.5": true, @@ -890,7 +248,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.43.0": { "0.43.2": true, @@ -903,18 +262,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.43.3": true, "0.43.4": true, "0.43.5": true, @@ -937,7 +284,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.43.1": { "0.43.2": true, @@ -950,18 +298,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.43.3": true, "0.43.4": true, "0.43.5": true, @@ -984,7 +320,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.43.2": { "0.43.2": true, @@ -997,18 +334,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.43.3": true, "0.43.4": true, "0.43.5": true, @@ -1031,7 +356,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.43.3": { "0.43.3": true, @@ -1045,18 +371,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.43.4": true, "0.43.5": true, "0.43.6": true, @@ -1078,7 +392,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.43.4": { "0.43.4": true, @@ -1093,18 +408,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.43.5": true, "0.43.6": true, "0.43.7": true, @@ -1125,7 +428,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.43.5": { "0.43.5": true, @@ -1141,18 +445,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.43.6": true, "0.43.7": true, "0.44.0": true, @@ -1172,7 +464,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.43.6": { "0.43.6": true, @@ -1189,18 +482,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.43.7": true, "0.44.0": true, "0.45.0": true, @@ -1219,7 +500,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.43.7": { "0.43.7": true, @@ -1237,18 +519,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.44.0": true, "0.45.0": true, "0.46.0": true, @@ -1266,7 +536,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.44.0": { "0.44.0": true, @@ -1285,18 +556,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.45.0": true, "0.46.0": true, "0.46.1": true, @@ -1313,7 +572,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.45.0": { "0.45.0": true, @@ -1333,18 +593,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.46.0": true, "0.46.1": true, "0.46.2": true, @@ -1360,7 +608,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.46.0": { "0.46.0": true, @@ -1381,18 +630,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.46.1": true, "0.46.2": true, "0.46.3": true, @@ -1407,7 +644,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.46.1": { "0.46.1": true, @@ -1429,18 +667,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.46.2": true, "0.46.3": true, "0.47.0": true, @@ -1454,7 +680,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.46.2": { "0.46.2": true, @@ -1477,18 +704,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.46.3": true, "0.47.0": true, "0.47.1": true, @@ -1501,7 +716,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.46.3": { "0.46.3": true, @@ -1525,18 +741,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.47.0": true, "0.47.1": true, "0.47.2": true, @@ -1548,7 +752,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.47.0": { "0.47.0": true, @@ -1573,18 +778,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.47.1": true, "0.47.2": true, "0.48.0": true, @@ -1595,7 +788,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.47.1": { "0.47.1": true, @@ -1621,18 +815,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.47.2": true, "0.48.0": true, "0.49.0": true, @@ -1642,7 +824,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.47.2": { "0.47.2": true, @@ -1669,18 +852,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.48.0": true, "0.49.0": true, "0.49.1": true, @@ -1689,7 +860,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.48.0": { "0.48.0": true, @@ -1717,18 +889,6 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.49.0": true, "0.49.1": true, "0.49.2": true, @@ -1736,7 +896,8 @@ "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.49.0": { "0.49.1": true, @@ -1766,24 +927,13 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.49.2": true, "0.50.0": true, "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.49.1": { "0.49.1": true, @@ -1813,24 +963,13 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.49.2": true, "0.50.0": true, "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.49.2": { "0.49.2": true, @@ -1861,23 +1000,12 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.50.0": true, "0.50.1": true, "0.51.0": false, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.50.0": { "0.50.0": true, @@ -1911,20 +1039,9 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.50.1": { "0.51.0": false, @@ -1958,20 +1075,9 @@ "0.40.2": true, "0.40.1": true, "0.40.0": true, - "0.39.0": true, - "0.38.1": true, - "0.38.0": true, - "0.37.0": true, - "0.36.0": true, - "0.35.0": true, - "0.34.1": true, - "0.34.0": true, - "0.33.0": true, - "0.32.0": true, - "0.31.0": true, - "0.30.0": true, "0.52.0": false, - "0.52.1": false + "0.52.1": false, + "0.53.0": false }, "0.51.0": { "0.51.0": true, @@ -2005,20 +1111,9 @@ "0.40.2": false, "0.40.1": false, "0.40.0": false, - "0.39.0": false, - "0.38.1": false, - "0.38.0": false, - "0.37.0": false, - "0.36.0": false, - "0.35.0": false, - "0.34.1": false, - "0.34.0": false, - "0.33.0": false, - "0.32.0": false, - "0.31.0": false, - "0.30.0": false, "0.52.0": true, - "0.52.1": true + "0.52.1": true, + "0.53.0": false }, "0.52.0": { "0.51.0": true, @@ -2052,20 +1147,9 @@ "0.40.2": false, "0.40.1": false, "0.40.0": false, - "0.39.0": false, - "0.38.1": false, - "0.38.0": false, - "0.37.0": false, - "0.36.0": false, - "0.35.0": false, - "0.34.1": false, - "0.34.0": false, - "0.33.0": false, - "0.32.0": false, - "0.31.0": false, - "0.30.0": false, "0.52.0": true, - "0.52.1": true + "0.52.1": true, + "0.53.0": false }, "0.52.1": { "0.51.0": true, @@ -2099,19 +1183,44 @@ "0.40.2": false, "0.40.1": false, "0.40.0": false, - "0.39.0": false, - "0.38.1": false, - "0.38.0": false, - "0.37.0": false, - "0.36.0": false, - "0.35.0": false, - "0.34.1": false, - "0.34.0": false, - "0.33.0": false, - "0.32.0": false, - "0.31.0": false, - "0.30.0": false, "0.52.0": true, - "0.52.1": true + "0.52.1": true, + "0.53.0": false + }, + "0.53.0": { + "0.51.0": false, + "0.50.1": false, + "0.50.0": false, + "0.49.2": false, + "0.49.1": false, + "0.49.0": false, + "0.48.0": false, + "0.47.2": false, + "0.47.1": false, + "0.47.0": false, + "0.46.3": false, + "0.46.2": false, + "0.46.1": false, + "0.46.0": false, + "0.45.0": false, + "0.44.0": false, + "0.43.7": false, + "0.43.6": false, + "0.43.5": false, + "0.43.4": false, + "0.43.3": false, + "0.43.2": false, + "0.43.1": false, + "0.43.0": false, + "0.42.1": false, + "0.42.0": false, + "0.41.1": false, + "0.41.0": false, + "0.40.2": false, + "0.40.1": false, + "0.40.0": false, + "0.52.0": false, + "0.52.1": false, + "0.53.0": true } } diff --git a/octue/resources/child.py b/octue/resources/child.py index 7ac6ec839..b0208472d 100644 --- a/octue/resources/child.py +++ b/octue/resources/child.py @@ -44,13 +44,13 @@ def __repr__(self): return f"<{type(self).__name__}({self.id!r})>" @property - def received_messages(self): - """Get the messages received from the child if it has been asked a question. If it hasn't, `None` is returned. + def received_events(self): + """Get the events received from the child if it has been asked a question. If it hasn't, `None` is returned. If an empty list is returned, no messages have been received. :return list(dict)|None: """ - return self._service.received_messages + return self._service.received_events def ask( self, @@ -60,29 +60,35 @@ def ask( subscribe_to_logs=True, allow_local_files=False, handle_monitor_message=None, - record_messages=True, + record_events=True, save_diagnostics="SAVE_DIAGNOSTICS_ON_CRASH", # This is repeated as a string here to avoid a circular import. question_uuid=None, + push_endpoint=None, + asynchronous=False, timeout=86400, maximum_heartbeat_interval=300, ): - """Ask the child a question and wait for its answer - i.e. send it input values and/or an input manifest and - wait for it to analyse them and return output values and/or an output manifest. The input values and manifest - must conform to the schema in the child's twine. - - :param any|None input_values: any input values for the question - :param octue.resources.manifest.Manifest|None input_manifest: an input manifest of any datasets needed for the question + """Ask the child either: + - A synchronous (ask-and-wait) question and wait for it to return an output. Questions are synchronous if + the `push_endpoint` isn't provided and `asynchronous=False`. + - An asynchronous (fire-and-forget) question and return immediately. To make a question asynchronous, provide + the `push_endpoint` argument or set `asynchronous=True`. + + :param any|None input_values: any input values for the question, conforming with the schema in the child's twine + :param octue.resources.manifest.Manifest|None input_manifest: an input manifest of any datasets needed for the question, conforming with the schema in the child's twine :param list(dict)|None children: a list of children for the child to use instead of its default children (if it uses children). These should be in the same format as in an app's app configuration file and have the same keys. :param bool subscribe_to_logs: if `True`, subscribe to logs from the child and handle them with the local log handlers :param bool allow_local_files: if `True`, allow the input manifest to contain references to local files - this should only be set to `True` if the child will have access to these local files :param callable|None handle_monitor_message: a function to handle monitor messages (e.g. send them to an endpoint for plotting or displaying) - this function should take a single JSON-compatible python primitive as an argument (note that this could be an array or object) - :param bool record_messages: if `True`, record messages received from the child in the `received_messages` property + :param bool record_events: if `True`, record messages received from the child in the `received_events` property :param str save_diagnostics: must be one of {"SAVE_DIAGNOSTICS_OFF", "SAVE_DIAGNOSTICS_ON_CRASH", "SAVE_DIAGNOSTICS_ON"}; if turned on, allow the input values and manifest (and its datasets) to be saved by the child either all the time or just if it fails while processing them :param str|None question_uuid: the UUID to use for the question if a specific one is needed; a UUID is generated if not + :param str|None push_endpoint: if answers to the question should be pushed to an endpoint, provide its URL here (the returned subscription will be a push subscription); if not, leave this as `None` + :param bool asynchronous: if `True`, don't wait for an answer or create an answer subscription (the result and other events can be retrieved from the event store later) :param float timeout: time in seconds to wait for an answer before raising a timeout error :param float|int maximum_heartbeat_interval: the maximum amount of time (in seconds) allowed between child heartbeats before an error is raised :raise TimeoutError: if the timeout is exceeded while waiting for an answer - :return dict: a dictionary containing the keys "output_values" and "output_manifest" + :return dict|octue.cloud.pub_sub.subscription.Subscription|None: for a synchronous question, a dictionary containing the keys "output_values" and "output_manifest" from the result; for a question with a push endpoint, the push subscription; for an asynchronous question, `None` """ subscription, _ = self._service.ask( service_id=self.id, @@ -93,13 +99,18 @@ def ask( allow_local_files=allow_local_files, save_diagnostics=save_diagnostics, question_uuid=question_uuid, + push_endpoint=push_endpoint, + asynchronous=asynchronous, timeout=timeout, ) + if push_endpoint or asynchronous: + return subscription + return self._service.wait_for_answer( subscription=subscription, handle_monitor_message=handle_monitor_message, - record_messages=record_messages, + record_events=record_events, timeout=timeout, maximum_heartbeat_interval=maximum_heartbeat_interval, ) diff --git a/octue/resources/dataset.py b/octue/resources/dataset.py index a2714ac94..ab0b55769 100644 --- a/octue/resources/dataset.py +++ b/octue/resources/dataset.py @@ -332,19 +332,17 @@ def get_file_by_label(self, label): return self.files.one(labels__contains=label) def download(self, local_directory=None): - """Download all files in the dataset into the given local directory. If no path to a local directory is given, - the files will be downloaded to temporary locations. + """Download all files in the dataset. - :param str|None local_directory: - :return None: + :param str|None local_directory: the path to a local directory to download the dataset into; if not provided, the files will be downloaded to a temporary directory + :return str: the absolute path to the local directory """ if not self.exists_in_cloud: raise CloudLocationNotSpecified( f"You can only download files from a cloud dataset. This dataset's path is {self.path!r}." ) - local_directory = local_directory or tempfile.TemporaryDirectory().name - + local_directory = os.path.abspath(local_directory or tempfile.TemporaryDirectory().name) datafiles_and_paths = [] for file in self.files: @@ -354,7 +352,7 @@ def download(self, local_directory=None): path_relative_to_dataset = self._datafile_path_relative_to_self(file, path_type="cloud_path") - local_path = os.path.abspath(os.path.join(local_directory, *path_relative_to_dataset.split("/"))) + local_path = os.path.join(local_directory, *path_relative_to_dataset.split("/")) datafiles_and_paths.append({"datafile": file, "local_path": local_path}) def download_datafile(datafile_and_path): @@ -371,6 +369,7 @@ def download_datafile(datafile_and_path): logger.debug("Downloaded datafile to %r.", path) logger.info("Downloaded %r to %r.", self, local_directory) + return local_directory def to_primitive(self, include_files=True): """Convert the dataset to a dictionary of primitives, converting its files into their paths for a lightweight diff --git a/octue/resources/manifest.py b/octue/resources/manifest.py index 2fc246315..e38f97623 100644 --- a/octue/resources/manifest.py +++ b/octue/resources/manifest.py @@ -66,6 +66,28 @@ def all_datasets_are_in_cloud(self): """ return all(dataset.all_files_are_in_cloud for dataset in self.datasets.values()) + def download(self, paths=None, download_all=True): + """Download all datasets in the manifest. + + :param dict|None paths: a mapping of dataset name to download directory path; if not provided, datasets are downloaded to temporary directories + :param bool download_all: if `False` and `paths` is provided, only download the datasets specified in `paths` + :return dict(str, str): the downloaded datasets mapped to the absolute paths of the directories they were downloaded into + """ + if paths is None: + download_all = True + paths = {} + + for name, dataset in self.datasets.items(): + download_path = paths.get(name) + + if not download_path and not download_all: + logger.info("%r dataset download skipped as its download path wasn't specified.", name) + continue + + paths[name] = dataset.download(local_directory=download_path) + + return paths + def update_dataset_paths(self, path_generator): """Update the path of each dataset according to the given path generator function. This method is thread-safe. diff --git a/octue/resources/service_backends.py b/octue/resources/service_backends.py index 9270b4a52..b0296ed21 100644 --- a/octue/resources/service_backends.py +++ b/octue/resources/service_backends.py @@ -36,17 +36,16 @@ class ServiceBackend(ABC): class GCPPubSubBackend(ServiceBackend): - """A dataclass containing the details needed to use Google Cloud Platform Pub/Sub as a Service backend. + """A dataclass containing the details needed to use Google Cloud Pub/Sub as a Service backend. - :param str project_name: + :param str project_name: the name of the project to use for Pub/Sub :return None: """ def __init__(self, project_name): if project_name is None: raise exceptions.CloudLocationNotSpecified( - "The project name must be specified for a service to connect to the correct Google Cloud Pub/Sub " - f"instance - it's currently {project_name!r}.", + "`project_name` must be specified for a service to connect to the correct service - received None." ) self.project_name = project_name diff --git a/octue/runner.py b/octue/runner.py index d1fac55d3..5228429f5 100644 --- a/octue/runner.py +++ b/octue/runner.py @@ -374,9 +374,7 @@ def wrapper(*args, **kwargs): try: return original_ask_method(**kwargs) finally: - self.diagnostics.add_question( - {"id": child.id, "key": key, **kwargs, "messages": child.received_messages} - ) + self.diagnostics.add_question({"id": child.id, "key": key, **kwargs, "events": child.received_events}) return wrapper diff --git a/octue/utils/dictionaries.py b/octue/utils/dictionaries.py new file mode 100644 index 000000000..6fee8c433 --- /dev/null +++ b/octue/utils/dictionaries.py @@ -0,0 +1,13 @@ +def make_minimal_dictionary(**kwargs): + """Make a dictionary with only the keyword arguments that have a non-`None` value. + + :param kwargs: any number of key-value pairs as keyword arguments + :return dict: a dictionary containing only the key-value pairs that have a non-`None` value + """ + data = {} + + for key, value in kwargs.items(): + if value is not None: + data[key] = value + + return data diff --git a/octue/utils/patches.py b/octue/utils/patches.py index 54883acff..b2196b62c 100644 --- a/octue/utils/patches.py +++ b/octue/utils/patches.py @@ -14,11 +14,25 @@ def __enter__(self): :return list(unittest.mock.MagicMock): """ - return [patch.start() for patch in self.patches] + return self.start() def __exit__(self, *args, **kwargs): """Stop the patches. + :return None: + """ + self.stop() + + def start(self): + """Start the patches and return the mocks they produce. + + :return list(unittest.mock.MagicMock): + """ + return [patch.start() for patch in self.patches] + + def stop(self): + """Stop the patches. + :return None: """ for patch in self.patches: diff --git a/octue/utils/testing.py b/octue/utils/testing.py index b1bc8deb6..270df0ed7 100644 --- a/octue/utils/testing.py +++ b/octue/utils/testing.py @@ -65,4 +65,4 @@ def _load_child_emulators(path): with open(os.path.join(path, "questions.json")) as f: questions = json.load(f) - return tuple(ChildEmulator(id=question["id"], messages=question["messages"]) for question in questions) + return tuple(ChildEmulator(id=question["id"], events=question["events"]) for question in questions) diff --git a/poetry.lock b/poetry.lock index 3185461e7..e91eec3fe 100644 --- a/poetry.lock +++ b/poetry.lock @@ -230,13 +230,13 @@ redis = ["redis (>=2.10.5)"] [[package]] name = "cachetools" -version = "5.3.2" +version = "5.3.3" description = "Extensible memoizing collections and decorators" optional = false python-versions = ">=3.7" files = [ - {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, - {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, + {file = "cachetools-5.3.3-py3-none-any.whl", hash = "sha256:0abad1021d3f8325b2fc1d2e9c8b9c9d57b04c3932657a72465447332c24d945"}, + {file = "cachetools-5.3.3.tar.gz", hash = "sha256:ba29e2dfa0b8b556606f097407ed1aa62080ee108ab0dc5ec9d6a723a007d105"}, ] [[package]] @@ -388,13 +388,13 @@ files = [ [[package]] name = "coolname" -version = "1.1.0" +version = "2.2.0" description = "Random name and slug generator" optional = false python-versions = "*" files = [ - {file = "coolname-1.1.0-py2.py3-none-any.whl", hash = "sha256:e6a83a0ac88640f4f3d2070438dbe112fe80cfebc119c93bd402976ec84c0978"}, - {file = "coolname-1.1.0.tar.gz", hash = "sha256:410fe6ea9999bf96f2856ef0c726d5f38782bbefb7bb1aca0e91e0dc98ed09e3"}, + {file = "coolname-2.2.0-py2.py3-none-any.whl", hash = "sha256:4d1563186cfaf71b394d5df4c744f8c41303b6846413645e31d31915cdeb13e8"}, + {file = "coolname-2.2.0.tar.gz", hash = "sha256:6c5d5731759104479e7ca195a9b64f7900ac5bead40183c09323c7d0be9e75c7"}, ] [[package]] @@ -501,6 +501,23 @@ calendars = ["convertdate", "convertdate", "hijri-converter"] fasttext = ["fasttext"] langdetect = ["langdetect"] +[[package]] +name = "db-dtypes" +version = "1.2.0" +description = "Pandas Data Types for SQL systems (BigQuery, Spanner)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "db-dtypes-1.2.0.tar.gz", hash = "sha256:3531bb1fb8b5fbab33121fe243ccc2ade16ab2524f4c113b05cc702a1908e6ea"}, + {file = "db_dtypes-1.2.0-py2.py3-none-any.whl", hash = "sha256:6320bddd31d096447ef749224d64aab00972ed20e4392d86f7d8b81ad79f7ff0"}, +] + +[package.dependencies] +numpy = ">=1.16.6" +packaging = ">=17.0" +pandas = ">=0.24.2" +pyarrow = ">=3.0.0" + [[package]] name = "dict2css" version = "0.3.0.post1" @@ -662,13 +679,13 @@ google-crc32c = "1.3.0" [[package]] name = "google-api-core" -version = "2.16.2" +version = "2.17.1" description = "Google API client core library" optional = false python-versions = ">=3.7" files = [ - {file = "google-api-core-2.16.2.tar.gz", hash = "sha256:032d37b45d1d6bdaf68fb11ff621e2593263a239fa9246e2e94325f9c47876d2"}, - {file = "google_api_core-2.16.2-py3-none-any.whl", hash = "sha256:449ca0e3f14c179b4165b664256066c7861610f70b6ffe54bb01a04e9b466929"}, + {file = "google-api-core-2.17.1.tar.gz", hash = "sha256:9df18a1f87ee0df0bc4eea2770ebc4228392d8cc4066655b320e2cfccb15db95"}, + {file = "google_api_core-2.17.1-py3-none-any.whl", hash = "sha256:610c5b90092c360736baccf17bd3efbcb30dd380e7a6dc28a71059edb8bd0d8e"}, ] [package.dependencies] @@ -692,13 +709,13 @@ grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] [[package]] name = "google-auth" -version = "2.27.0" +version = "2.28.1" description = "Google Authentication Library" optional = false python-versions = ">=3.7" files = [ - {file = "google-auth-2.27.0.tar.gz", hash = "sha256:e863a56ccc2d8efa83df7a80272601e43487fa9a728a376205c86c26aaefa821"}, - {file = "google_auth-2.27.0-py2.py3-none-any.whl", hash = "sha256:8e4bad367015430ff253fe49d500fdc3396c1a434db5740828c728e45bcce245"}, + {file = "google-auth-2.28.1.tar.gz", hash = "sha256:34fc3046c257cedcf1622fc4b31fc2be7923d9b4d44973d481125ecc50d83885"}, + {file = "google_auth-2.28.1-py2.py3-none-any.whl", hash = "sha256:25141e2d7a14bfcba945f5e9827f98092716e99482562f15306e5b026e21aa72"}, ] [package.dependencies] @@ -713,6 +730,36 @@ pyopenssl = ["cryptography (>=38.0.3)", "pyopenssl (>=20.0.0)"] reauth = ["pyu2f (>=0.1.5)"] requests = ["requests (>=2.20.0,<3.0.0.dev0)"] +[[package]] +name = "google-cloud-bigquery" +version = "3.18.0" +description = "Google BigQuery API client library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-cloud-bigquery-3.18.0.tar.gz", hash = "sha256:74f0fc6f0ba9477f808d25924dc8a052c55f7ca91064e83e16d3ee5fb7ca77ab"}, + {file = "google_cloud_bigquery-3.18.0-py2.py3-none-any.whl", hash = "sha256:3520552075502c69710d37b1e9600c84e6974ad271914677d16bfafa502857fb"}, +] + +[package.dependencies] +google-api-core = ">=1.31.5,<2.0.dev0 || >2.3.0,<3.0.0dev" +google-cloud-core = ">=1.6.0,<3.0.0dev" +google-resumable-media = ">=0.6.0,<3.0dev" +packaging = ">=20.0.0" +python-dateutil = ">=2.7.2,<3.0dev" +requests = ">=2.21.0,<3.0.0dev" + +[package.extras] +all = ["Shapely (>=1.8.4,<3.0.0dev)", "db-dtypes (>=0.3.0,<2.0.0dev)", "geopandas (>=0.9.0,<1.0dev)", "google-cloud-bigquery-storage (>=2.6.0,<3.0.0dev)", "grpcio (>=1.47.0,<2.0dev)", "grpcio (>=1.49.1,<2.0dev)", "importlib-metadata (>=1.0.0)", "ipykernel (>=6.0.0)", "ipython (>=7.23.1,!=8.1.0)", "ipywidgets (>=7.7.0)", "opentelemetry-api (>=1.1.0)", "opentelemetry-instrumentation (>=0.20b0)", "opentelemetry-sdk (>=1.1.0)", "pandas (>=1.1.0)", "proto-plus (>=1.15.0,<2.0.0dev)", "protobuf (>=3.19.5,!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev)", "pyarrow (>=3.0.0)", "tqdm (>=4.7.4,<5.0.0dev)"] +bigquery-v2 = ["proto-plus (>=1.15.0,<2.0.0dev)", "protobuf (>=3.19.5,!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev)"] +bqstorage = ["google-cloud-bigquery-storage (>=2.6.0,<3.0.0dev)", "grpcio (>=1.47.0,<2.0dev)", "grpcio (>=1.49.1,<2.0dev)", "pyarrow (>=3.0.0)"] +geopandas = ["Shapely (>=1.8.4,<3.0.0dev)", "geopandas (>=0.9.0,<1.0dev)"] +ipython = ["ipykernel (>=6.0.0)", "ipython (>=7.23.1,!=8.1.0)"] +ipywidgets = ["ipykernel (>=6.0.0)", "ipywidgets (>=7.7.0)"] +opentelemetry = ["opentelemetry-api (>=1.1.0)", "opentelemetry-instrumentation (>=0.20b0)", "opentelemetry-sdk (>=1.1.0)"] +pandas = ["db-dtypes (>=0.3.0,<2.0.0dev)", "importlib-metadata (>=1.0.0)", "pandas (>=1.1.0)", "pyarrow (>=3.0.0)"] +tqdm = ["tqdm (>=4.7.4,<5.0.0dev)"] + [[package]] name = "google-cloud-core" version = "2.4.1" @@ -734,17 +781,18 @@ grpc = ["grpcio (>=1.38.0,<2.0dev)", "grpcio-status (>=1.38.0,<2.0.dev0)"] [[package]] name = "google-cloud-pubsub" -version = "2.19.1" +version = "2.20.0" description = "Google Cloud Pub/Sub API client library" optional = false python-versions = ">=3.7" files = [ - {file = "google-cloud-pubsub-2.19.1.tar.gz", hash = "sha256:c10d95ebaf903f923b14aa8ec5b7c80916138edf299c65a1c1a9431fd5665d25"}, - {file = "google_cloud_pubsub-2.19.1-py2.py3-none-any.whl", hash = "sha256:602eacbe5735f38d3ae384ece8f235db0594de78ad718a35fcb948dc4964612c"}, + {file = "google-cloud-pubsub-2.20.0.tar.gz", hash = "sha256:48c8e17a8168c43e3188635cbd9e07fbe3004120433712ce84b3a04bbf18c188"}, + {file = "google_cloud_pubsub-2.20.0-py2.py3-none-any.whl", hash = "sha256:8c69ed04800f4f552cdf3b9028f06d9271ac6e60443b2308c984def442e69684"}, ] [package.dependencies] google-api-core = {version = ">=1.34.0,<2.0.dev0 || >=2.11.dev0,<3.0.0dev", extras = ["grpc"]} +google-auth = ">=2.14.1,<3.0.0dev" grpc-google-iam-v1 = ">=0.12.4,<1.0.0dev" grpcio = ">=1.51.3,<2.0dev" grpcio-status = ">=1.33.2" @@ -759,35 +807,36 @@ libcst = ["libcst (>=0.3.10)"] [[package]] name = "google-cloud-secret-manager" -version = "2.18.0" +version = "2.18.3" description = "Google Cloud Secret Manager API client library" optional = false python-versions = ">=3.7" files = [ - {file = "google-cloud-secret-manager-2.18.0.tar.gz", hash = "sha256:4e499bd33ff7aeff271bd67487e21d5404297a86dc4873ee861d637b01b30b4e"}, - {file = "google_cloud_secret_manager-2.18.0-py2.py3-none-any.whl", hash = "sha256:06ce001648a75fc7e992e3e658772b10a6ebc738a49463204c44409137ad5890"}, + {file = "google-cloud-secret-manager-2.18.3.tar.gz", hash = "sha256:1db2f409324536e34f985081d389e3974ca3a3668df7845cad0be03ab8c0fa7d"}, + {file = "google_cloud_secret_manager-2.18.3-py2.py3-none-any.whl", hash = "sha256:4d4af82bddd9099ebdbe79e0c6b68f6c6cabea8323a3c1275bcead8f56310fb7"}, ] [package.dependencies] -google-api-core = {version = ">=1.34.0,<2.0.dev0 || >=2.11.dev0,<3.0.0dev", extras = ["grpc"]} +google-api-core = {version = ">=1.34.1,<2.0.dev0 || >=2.11.dev0,<3.0.0dev", extras = ["grpc"]} +google-auth = ">=2.14.1,<2.24.0 || >2.24.0,<2.25.0 || >2.25.0,<3.0.0dev" grpc-google-iam-v1 = ">=0.12.4,<1.0.0dev" proto-plus = ">=1.22.3,<2.0.0dev" protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev" [[package]] name = "google-cloud-storage" -version = "2.14.0" +version = "2.15.0" description = "Google Cloud Storage API client library" optional = false python-versions = ">=3.7" files = [ - {file = "google-cloud-storage-2.14.0.tar.gz", hash = "sha256:2d23fcf59b55e7b45336729c148bb1c464468c69d5efbaee30f7201dd90eb97e"}, - {file = "google_cloud_storage-2.14.0-py2.py3-none-any.whl", hash = "sha256:8641243bbf2a2042c16a6399551fbb13f062cbc9a2de38d6c0bb5426962e9dbd"}, + {file = "google-cloud-storage-2.15.0.tar.gz", hash = "sha256:7560a3c48a03d66c553dc55215d35883c680fe0ab44c23aa4832800ccc855c74"}, + {file = "google_cloud_storage-2.15.0-py2.py3-none-any.whl", hash = "sha256:5d9237f88b648e1d724a0f20b5cde65996a37fe51d75d17660b1404097327dd2"}, ] [package.dependencies] -google-api-core = ">=1.31.5,<2.0.dev0 || >2.3.0,<3.0.0dev" -google-auth = ">=2.23.3,<3.0dev" +google-api-core = ">=2.15.0,<3.0.0dev" +google-auth = ">=2.26.1,<3.0dev" google-cloud-core = ">=2.3.0,<3.0dev" google-crc32c = ">=1.0,<2.0dev" google-resumable-media = ">=2.6.0" @@ -905,84 +954,84 @@ protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4 [[package]] name = "grpcio" -version = "1.60.1" +version = "1.62.0" description = "HTTP/2-based RPC framework" optional = false python-versions = ">=3.7" files = [ - {file = "grpcio-1.60.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:14e8f2c84c0832773fb3958240c69def72357bc11392571f87b2d7b91e0bb092"}, - {file = "grpcio-1.60.1-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:33aed0a431f5befeffd9d346b0fa44b2c01aa4aeae5ea5b2c03d3e25e0071216"}, - {file = "grpcio-1.60.1-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:fead980fbc68512dfd4e0c7b1f5754c2a8e5015a04dea454b9cada54a8423525"}, - {file = "grpcio-1.60.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:082081e6a36b6eb5cf0fd9a897fe777dbb3802176ffd08e3ec6567edd85bc104"}, - {file = "grpcio-1.60.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:55ccb7db5a665079d68b5c7c86359ebd5ebf31a19bc1a91c982fd622f1e31ff2"}, - {file = "grpcio-1.60.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:9b54577032d4f235452f77a83169b6527bf4b77d73aeada97d45b2aaf1bf5ce0"}, - {file = "grpcio-1.60.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7d142bcd604166417929b071cd396aa13c565749a4c840d6c702727a59d835eb"}, - {file = "grpcio-1.60.1-cp310-cp310-win32.whl", hash = "sha256:2a6087f234cb570008a6041c8ffd1b7d657b397fdd6d26e83d72283dae3527b1"}, - {file = "grpcio-1.60.1-cp310-cp310-win_amd64.whl", hash = "sha256:f2212796593ad1d0235068c79836861f2201fc7137a99aa2fea7beeb3b101177"}, - {file = "grpcio-1.60.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:79ae0dc785504cb1e1788758c588c711f4e4a0195d70dff53db203c95a0bd303"}, - {file = "grpcio-1.60.1-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:4eec8b8c1c2c9b7125508ff7c89d5701bf933c99d3910e446ed531cd16ad5d87"}, - {file = "grpcio-1.60.1-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:8c9554ca8e26241dabe7951aa1fa03a1ba0856688ecd7e7bdbdd286ebc272e4c"}, - {file = "grpcio-1.60.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:91422ba785a8e7a18725b1dc40fbd88f08a5bb4c7f1b3e8739cab24b04fa8a03"}, - {file = "grpcio-1.60.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cba6209c96828711cb7c8fcb45ecef8c8859238baf15119daa1bef0f6c84bfe7"}, - {file = "grpcio-1.60.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c71be3f86d67d8d1311c6076a4ba3b75ba5703c0b856b4e691c9097f9b1e8bd2"}, - {file = "grpcio-1.60.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:af5ef6cfaf0d023c00002ba25d0751e5995fa0e4c9eec6cd263c30352662cbce"}, - {file = "grpcio-1.60.1-cp311-cp311-win32.whl", hash = "sha256:a09506eb48fa5493c58f946c46754ef22f3ec0df64f2b5149373ff31fb67f3dd"}, - {file = "grpcio-1.60.1-cp311-cp311-win_amd64.whl", hash = "sha256:49c9b6a510e3ed8df5f6f4f3c34d7fbf2d2cae048ee90a45cd7415abab72912c"}, - {file = "grpcio-1.60.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:b58b855d0071575ea9c7bc0d84a06d2edfbfccec52e9657864386381a7ce1ae9"}, - {file = "grpcio-1.60.1-cp312-cp312-macosx_10_10_universal2.whl", hash = "sha256:a731ac5cffc34dac62053e0da90f0c0b8560396a19f69d9703e88240c8f05858"}, - {file = "grpcio-1.60.1-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:cf77f8cf2a651fbd869fbdcb4a1931464189cd210abc4cfad357f1cacc8642a6"}, - {file = "grpcio-1.60.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c557e94e91a983e5b1e9c60076a8fd79fea1e7e06848eb2e48d0ccfb30f6e073"}, - {file = "grpcio-1.60.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:069fe2aeee02dfd2135d562d0663fe70fbb69d5eed6eb3389042a7e963b54de8"}, - {file = "grpcio-1.60.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:cb0af13433dbbd1c806e671d81ec75bd324af6ef75171fd7815ca3074fe32bfe"}, - {file = "grpcio-1.60.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2f44c32aef186bbba254129cea1df08a20be414144ac3bdf0e84b24e3f3b2e05"}, - {file = "grpcio-1.60.1-cp312-cp312-win32.whl", hash = "sha256:a212e5dea1a4182e40cd3e4067ee46be9d10418092ce3627475e995cca95de21"}, - {file = "grpcio-1.60.1-cp312-cp312-win_amd64.whl", hash = "sha256:6e490fa5f7f5326222cb9f0b78f207a2b218a14edf39602e083d5f617354306f"}, - {file = "grpcio-1.60.1-cp37-cp37m-linux_armv7l.whl", hash = "sha256:4216e67ad9a4769117433814956031cb300f85edc855252a645a9a724b3b6594"}, - {file = "grpcio-1.60.1-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:73e14acd3d4247169955fae8fb103a2b900cfad21d0c35f0dcd0fdd54cd60367"}, - {file = "grpcio-1.60.1-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:6ecf21d20d02d1733e9c820fb5c114c749d888704a7ec824b545c12e78734d1c"}, - {file = "grpcio-1.60.1-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:33bdea30dcfd4f87b045d404388469eb48a48c33a6195a043d116ed1b9a0196c"}, - {file = "grpcio-1.60.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53b69e79d00f78c81eecfb38f4516080dc7f36a198b6b37b928f1c13b3c063e9"}, - {file = "grpcio-1.60.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:39aa848794b887120b1d35b1b994e445cc028ff602ef267f87c38122c1add50d"}, - {file = "grpcio-1.60.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:72153a0d2e425f45b884540a61c6639436ddafa1829a42056aa5764b84108b8e"}, - {file = "grpcio-1.60.1-cp37-cp37m-win_amd64.whl", hash = "sha256:50d56280b482875d1f9128ce596e59031a226a8b84bec88cb2bf76c289f5d0de"}, - {file = "grpcio-1.60.1-cp38-cp38-linux_armv7l.whl", hash = "sha256:6d140bdeb26cad8b93c1455fa00573c05592793c32053d6e0016ce05ba267549"}, - {file = "grpcio-1.60.1-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:bc808924470643b82b14fe121923c30ec211d8c693e747eba8a7414bc4351a23"}, - {file = "grpcio-1.60.1-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:70c83bb530572917be20c21f3b6be92cd86b9aecb44b0c18b1d3b2cc3ae47df0"}, - {file = "grpcio-1.60.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9b106bc52e7f28170e624ba61cc7dc6829566e535a6ec68528f8e1afbed1c41f"}, - {file = "grpcio-1.60.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:30e980cd6db1088c144b92fe376747328d5554bc7960ce583ec7b7d81cd47287"}, - {file = "grpcio-1.60.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:0c5807e9152eff15f1d48f6b9ad3749196f79a4a050469d99eecb679be592acc"}, - {file = "grpcio-1.60.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f1c3dc536b3ee124e8b24feb7533e5c70b9f2ef833e3b2e5513b2897fd46763a"}, - {file = "grpcio-1.60.1-cp38-cp38-win32.whl", hash = "sha256:d7404cebcdb11bb5bd40bf94131faf7e9a7c10a6c60358580fe83913f360f929"}, - {file = "grpcio-1.60.1-cp38-cp38-win_amd64.whl", hash = "sha256:c8754c75f55781515a3005063d9a05878b2cfb3cb7e41d5401ad0cf19de14872"}, - {file = "grpcio-1.60.1-cp39-cp39-linux_armv7l.whl", hash = "sha256:0250a7a70b14000fa311de04b169cc7480be6c1a769b190769d347939d3232a8"}, - {file = "grpcio-1.60.1-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:660fc6b9c2a9ea3bb2a7e64ba878c98339abaf1811edca904ac85e9e662f1d73"}, - {file = "grpcio-1.60.1-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:76eaaba891083fcbe167aa0f03363311a9f12da975b025d30e94b93ac7a765fc"}, - {file = "grpcio-1.60.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e5d97c65ea7e097056f3d1ead77040ebc236feaf7f71489383d20f3b4c28412a"}, - {file = "grpcio-1.60.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bb2a2911b028f01c8c64d126f6b632fcd8a9ac975aa1b3855766c94e4107180"}, - {file = "grpcio-1.60.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:5a1ebbae7e2214f51b1f23b57bf98eeed2cf1ba84e4d523c48c36d5b2f8829ff"}, - {file = "grpcio-1.60.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:9a66f4d2a005bc78e61d805ed95dedfcb35efa84b7bba0403c6d60d13a3de2d6"}, - {file = "grpcio-1.60.1-cp39-cp39-win32.whl", hash = "sha256:8d488fbdbf04283f0d20742b64968d44825617aa6717b07c006168ed16488804"}, - {file = "grpcio-1.60.1-cp39-cp39-win_amd64.whl", hash = "sha256:61b7199cd2a55e62e45bfb629a35b71fc2c0cb88f686a047f25b1112d3810904"}, - {file = "grpcio-1.60.1.tar.gz", hash = "sha256:dd1d3a8d1d2e50ad9b59e10aa7f07c7d1be2b367f3f2d33c5fade96ed5460962"}, + {file = "grpcio-1.62.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:136ffd79791b1eddda8d827b607a6285474ff8a1a5735c4947b58c481e5e4271"}, + {file = "grpcio-1.62.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:d6a56ba703be6b6267bf19423d888600c3f574ac7c2cc5e6220af90662a4d6b0"}, + {file = "grpcio-1.62.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:4cd356211579043fce9f52acc861e519316fff93980a212c8109cca8f47366b6"}, + {file = "grpcio-1.62.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e803e9b58d8f9b4ff0ea991611a8d51b31c68d2e24572cd1fe85e99e8cc1b4f8"}, + {file = "grpcio-1.62.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f4c04fe33039b35b97c02d2901a164bbbb2f21fb9c4e2a45a959f0b044c3512c"}, + {file = "grpcio-1.62.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:95370c71b8c9062f9ea033a0867c4c73d6f0ff35113ebd2618171ec1f1e903e0"}, + {file = "grpcio-1.62.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c912688acc05e4ff012c8891803659d6a8a8b5106f0f66e0aed3fb7e77898fa6"}, + {file = "grpcio-1.62.0-cp310-cp310-win32.whl", hash = "sha256:821a44bd63d0f04e33cf4ddf33c14cae176346486b0df08b41a6132b976de5fc"}, + {file = "grpcio-1.62.0-cp310-cp310-win_amd64.whl", hash = "sha256:81531632f93fece32b2762247c4c169021177e58e725494f9a746ca62c83acaa"}, + {file = "grpcio-1.62.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:3fa15850a6aba230eed06b236287c50d65a98f05054a0f01ccedf8e1cc89d57f"}, + {file = "grpcio-1.62.0-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:36df33080cd7897623feff57831eb83c98b84640b016ce443305977fac7566fb"}, + {file = "grpcio-1.62.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:7a195531828b46ea9c4623c47e1dc45650fc7206f8a71825898dd4c9004b0928"}, + {file = "grpcio-1.62.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ab140a3542bbcea37162bdfc12ce0d47a3cda3f2d91b752a124cc9fe6776a9e2"}, + {file = "grpcio-1.62.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f9d6c3223914abb51ac564dc9c3782d23ca445d2864321b9059d62d47144021"}, + {file = "grpcio-1.62.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:fbe0c20ce9a1cff75cfb828b21f08d0a1ca527b67f2443174af6626798a754a4"}, + {file = "grpcio-1.62.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:38f69de9c28c1e7a8fd24e4af4264726637b72f27c2099eaea6e513e7142b47e"}, + {file = "grpcio-1.62.0-cp311-cp311-win32.whl", hash = "sha256:ce1aafdf8d3f58cb67664f42a617af0e34555fe955450d42c19e4a6ad41c84bd"}, + {file = "grpcio-1.62.0-cp311-cp311-win_amd64.whl", hash = "sha256:eef1d16ac26c5325e7d39f5452ea98d6988c700c427c52cbc7ce3201e6d93334"}, + {file = "grpcio-1.62.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:8aab8f90b2a41208c0a071ec39a6e5dbba16fd827455aaa070fec241624ccef8"}, + {file = "grpcio-1.62.0-cp312-cp312-macosx_10_10_universal2.whl", hash = "sha256:62aa1659d8b6aad7329ede5d5b077e3d71bf488d85795db517118c390358d5f6"}, + {file = "grpcio-1.62.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:0d7ae7fc7dbbf2d78d6323641ded767d9ec6d121aaf931ec4a5c50797b886532"}, + {file = "grpcio-1.62.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f359d635ee9428f0294bea062bb60c478a8ddc44b0b6f8e1f42997e5dc12e2ee"}, + {file = "grpcio-1.62.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77d48e5b1f8f4204889f1acf30bb57c30378e17c8d20df5acbe8029e985f735c"}, + {file = "grpcio-1.62.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:662d3df5314ecde3184cf87ddd2c3a66095b3acbb2d57a8cada571747af03873"}, + {file = "grpcio-1.62.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:92cdb616be44c8ac23a57cce0243af0137a10aa82234f23cd46e69e115071388"}, + {file = "grpcio-1.62.0-cp312-cp312-win32.whl", hash = "sha256:0b9179478b09ee22f4a36b40ca87ad43376acdccc816ce7c2193a9061bf35701"}, + {file = "grpcio-1.62.0-cp312-cp312-win_amd64.whl", hash = "sha256:614c3ed234208e76991992342bab725f379cc81c7dd5035ee1de2f7e3f7a9842"}, + {file = "grpcio-1.62.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:7e1f51e2a460b7394670fdb615e26d31d3260015154ea4f1501a45047abe06c9"}, + {file = "grpcio-1.62.0-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:bcff647e7fe25495e7719f779cc219bbb90b9e79fbd1ce5bda6aae2567f469f2"}, + {file = "grpcio-1.62.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:56ca7ba0b51ed0de1646f1735154143dcbdf9ec2dbe8cc6645def299bb527ca1"}, + {file = "grpcio-1.62.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e84bfb2a734e4a234b116be208d6f0214e68dcf7804306f97962f93c22a1839"}, + {file = "grpcio-1.62.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c1488b31a521fbba50ae86423f5306668d6f3a46d124f7819c603979fc538c4"}, + {file = "grpcio-1.62.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:98d8f4eb91f1ce0735bf0b67c3b2a4fea68b52b2fd13dc4318583181f9219b4b"}, + {file = "grpcio-1.62.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:b3d3d755cfa331d6090e13aac276d4a3fb828bf935449dc16c3d554bf366136b"}, + {file = "grpcio-1.62.0-cp37-cp37m-win_amd64.whl", hash = "sha256:a33f2bfd8a58a02aab93f94f6c61279be0f48f99fcca20ebaee67576cd57307b"}, + {file = "grpcio-1.62.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:5e709f7c8028ce0443bddc290fb9c967c1e0e9159ef7a030e8c21cac1feabd35"}, + {file = "grpcio-1.62.0-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:2f3d9a4d0abb57e5f49ed5039d3ed375826c2635751ab89dcc25932ff683bbb6"}, + {file = "grpcio-1.62.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:62ccb92f594d3d9fcd00064b149a0187c246b11e46ff1b7935191f169227f04c"}, + {file = "grpcio-1.62.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:921148f57c2e4b076af59a815467d399b7447f6e0ee10ef6d2601eb1e9c7f402"}, + {file = "grpcio-1.62.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f897b16190b46bc4d4aaf0a32a4b819d559a37a756d7c6b571e9562c360eed72"}, + {file = "grpcio-1.62.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1bc8449084fe395575ed24809752e1dc4592bb70900a03ca42bf236ed5bf008f"}, + {file = "grpcio-1.62.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:81d444e5e182be4c7856cd33a610154fe9ea1726bd071d07e7ba13fafd202e38"}, + {file = "grpcio-1.62.0-cp38-cp38-win32.whl", hash = "sha256:88f41f33da3840b4a9bbec68079096d4caf629e2c6ed3a72112159d570d98ebe"}, + {file = "grpcio-1.62.0-cp38-cp38-win_amd64.whl", hash = "sha256:fc2836cb829895ee190813446dce63df67e6ed7b9bf76060262c55fcd097d270"}, + {file = "grpcio-1.62.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:fcc98cff4084467839d0a20d16abc2a76005f3d1b38062464d088c07f500d170"}, + {file = "grpcio-1.62.0-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:0d3dee701e48ee76b7d6fbbba18ba8bc142e5b231ef7d3d97065204702224e0e"}, + {file = "grpcio-1.62.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:b7a6be562dd18e5d5bec146ae9537f20ae1253beb971c0164f1e8a2f5a27e829"}, + {file = "grpcio-1.62.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:29cb592c4ce64a023712875368bcae13938c7f03e99f080407e20ffe0a9aa33b"}, + {file = "grpcio-1.62.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1eda79574aec8ec4d00768dcb07daba60ed08ef32583b62b90bbf274b3c279f7"}, + {file = "grpcio-1.62.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7eea57444a354ee217fda23f4b479a4cdfea35fb918ca0d8a0e73c271e52c09c"}, + {file = "grpcio-1.62.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0e97f37a3b7c89f9125b92d22e9c8323f4e76e7993ba7049b9f4ccbe8bae958a"}, + {file = "grpcio-1.62.0-cp39-cp39-win32.whl", hash = "sha256:39cd45bd82a2e510e591ca2ddbe22352e8413378852ae814549c162cf3992a93"}, + {file = "grpcio-1.62.0-cp39-cp39-win_amd64.whl", hash = "sha256:b71c65427bf0ec6a8b48c68c17356cb9fbfc96b1130d20a07cb462f4e4dcdcd5"}, + {file = "grpcio-1.62.0.tar.gz", hash = "sha256:748496af9238ac78dcd98cce65421f1adce28c3979393e3609683fcd7f3880d7"}, ] [package.extras] -protobuf = ["grpcio-tools (>=1.60.1)"] +protobuf = ["grpcio-tools (>=1.62.0)"] [[package]] name = "grpcio-status" -version = "1.60.1" +version = "1.62.0" description = "Status proto mapping for gRPC" optional = false python-versions = ">=3.6" files = [ - {file = "grpcio-status-1.60.1.tar.gz", hash = "sha256:61b5aab8989498e8aa142c20b88829ea5d90d18c18c853b9f9e6d407d37bf8b4"}, - {file = "grpcio_status-1.60.1-py3-none-any.whl", hash = "sha256:3034fdb239185b6e0f3169d08c268c4507481e4b8a434c21311a03d9eb5889a0"}, + {file = "grpcio-status-1.62.0.tar.gz", hash = "sha256:0d693e9c09880daeaac060d0c3dba1ae470a43c99e5d20dfeafd62cf7e08a85d"}, + {file = "grpcio_status-1.62.0-py3-none-any.whl", hash = "sha256:3baac03fcd737310e67758c4082a188107f771d32855bce203331cd4c9aa687a"}, ] [package.dependencies] googleapis-common-protos = ">=1.5.5" -grpcio = ">=1.60.1" +grpcio = ">=1.62.0" protobuf = ">=4.21.6" [[package]] @@ -1629,6 +1678,43 @@ files = [ {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, ] +[[package]] +name = "pyarrow" +version = "12.0.1" +description = "Python library for Apache Arrow" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pyarrow-12.0.1-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:6d288029a94a9bb5407ceebdd7110ba398a00412c5b0155ee9813a40d246c5df"}, + {file = "pyarrow-12.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:345e1828efdbd9aa4d4de7d5676778aba384a2c3add896d995b23d368e60e5af"}, + {file = "pyarrow-12.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8d6009fdf8986332b2169314da482baed47ac053311c8934ac6651e614deacd6"}, + {file = "pyarrow-12.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d3c4cbbf81e6dd23fe921bc91dc4619ea3b79bc58ef10bce0f49bdafb103daf"}, + {file = "pyarrow-12.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:cdacf515ec276709ac8042c7d9bd5be83b4f5f39c6c037a17a60d7ebfd92c890"}, + {file = "pyarrow-12.0.1-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:749be7fd2ff260683f9cc739cb862fb11be376de965a2a8ccbf2693b098db6c7"}, + {file = "pyarrow-12.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6895b5fb74289d055c43db3af0de6e16b07586c45763cb5e558d38b86a91e3a7"}, + {file = "pyarrow-12.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1887bdae17ec3b4c046fcf19951e71b6a619f39fa674f9881216173566c8f718"}, + {file = "pyarrow-12.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e2c9cb8eeabbadf5fcfc3d1ddea616c7ce893db2ce4dcef0ac13b099ad7ca082"}, + {file = "pyarrow-12.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:ce4aebdf412bd0eeb800d8e47db854f9f9f7e2f5a0220440acf219ddfddd4f63"}, + {file = "pyarrow-12.0.1-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:e0d8730c7f6e893f6db5d5b86eda42c0a130842d101992b581e2138e4d5663d3"}, + {file = "pyarrow-12.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:43364daec02f69fec89d2315f7fbfbeec956e0d991cbbef471681bd77875c40f"}, + {file = "pyarrow-12.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:051f9f5ccf585f12d7de836e50965b3c235542cc896959320d9776ab93f3b33d"}, + {file = "pyarrow-12.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:be2757e9275875d2a9c6e6052ac7957fbbfc7bc7370e4a036a9b893e96fedaba"}, + {file = "pyarrow-12.0.1-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:cf812306d66f40f69e684300f7af5111c11f6e0d89d6b733e05a3de44961529d"}, + {file = "pyarrow-12.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:459a1c0ed2d68671188b2118c63bac91eaef6fc150c77ddd8a583e3c795737bf"}, + {file = "pyarrow-12.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:85e705e33eaf666bbe508a16fd5ba27ca061e177916b7a317ba5a51bee43384c"}, + {file = "pyarrow-12.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9120c3eb2b1f6f516a3b7a9714ed860882d9ef98c4b17edcdc91d95b7528db60"}, + {file = "pyarrow-12.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:c780f4dc40460015d80fcd6a6140de80b615349ed68ef9adb653fe351778c9b3"}, + {file = "pyarrow-12.0.1-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:a3c63124fc26bf5f95f508f5d04e1ece8cc23a8b0af2a1e6ab2b1ec3fdc91b24"}, + {file = "pyarrow-12.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b13329f79fa4472324f8d32dc1b1216616d09bd1e77cfb13104dec5463632c36"}, + {file = "pyarrow-12.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb656150d3d12ec1396f6dde542db1675a95c0cc8366d507347b0beed96e87ca"}, + {file = "pyarrow-12.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6251e38470da97a5b2e00de5c6a049149f7b2bd62f12fa5dbb9ac674119ba71a"}, + {file = "pyarrow-12.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:3de26da901216149ce086920547dfff5cd22818c9eab67ebc41e863a5883bac7"}, + {file = "pyarrow-12.0.1.tar.gz", hash = "sha256:cce317fc96e5b71107bf1f9f184d5e54e2bd14bbf3f9a3d62819961f0af86fec"}, +] + +[package.dependencies] +numpy = ">=1.16.6" + [[package]] name = "pyasn1" version = "0.5.1" @@ -1752,13 +1838,13 @@ testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "no [[package]] name = "python-dateutil" -version = "2.8.2" +version = "2.9.0.post0" description = "Extensions to the standard Python datetime module" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ - {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, - {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, + {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, + {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, ] [package.dependencies] @@ -1968,13 +2054,13 @@ pyasn1 = ">=0.1.3" [[package]] name = "ruamel-yaml" -version = "0.18.5" +version = "0.18.6" description = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order" optional = false python-versions = ">=3.7" files = [ - {file = "ruamel.yaml-0.18.5-py3-none-any.whl", hash = "sha256:a013ac02f99a69cdd6277d9664689eb1acba07069f912823177c5eced21a6ada"}, - {file = "ruamel.yaml-0.18.5.tar.gz", hash = "sha256:61917e3a35a569c1133a8f772e1226961bf5a1198bea7e23f06a0841dea1ab0e"}, + {file = "ruamel.yaml-0.18.6-py3-none-any.whl", hash = "sha256:57b53ba33def16c4f3d807c0ccbc00f8a6081827e81ba2491691b76882d0c636"}, + {file = "ruamel.yaml-0.18.6.tar.gz", hash = "sha256:8b27e6a217e786c6fbe5634d8f3f11bc63e0f80f6a5890f28863d9c45aac311b"}, ] [package.dependencies] @@ -2505,13 +2591,13 @@ typing-extensions = ">=3.7.4" [[package]] name = "tzdata" -version = "2023.4" +version = "2024.1" description = "Provider of IANA time zone data" optional = false python-versions = ">=2" files = [ - {file = "tzdata-2023.4-py2.py3-none-any.whl", hash = "sha256:aa3ace4329eeacda5b7beb7ea08ece826c28d761cda36e747cfbf97996d39bf3"}, - {file = "tzdata-2023.4.tar.gz", hash = "sha256:dd54c94f294765522c77399649b4fefd95522479a664a0cec87f41bebc6148c9"}, + {file = "tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"}, + {file = "tzdata-2024.1.tar.gz", hash = "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd"}, ] [[package]] @@ -2551,13 +2637,13 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "virtualenv" -version = "20.25.0" +version = "20.25.1" description = "Virtual Python Environment builder" optional = false python-versions = ">=3.7" files = [ - {file = "virtualenv-20.25.0-py3-none-any.whl", hash = "sha256:4238949c5ffe6876362d9c0180fc6c3a824a7b12b80604eeb8085f2ed7460de3"}, - {file = "virtualenv-20.25.0.tar.gz", hash = "sha256:bf51c0d9c7dd63ea8e44086fa1e4fb1093a31e963b86959257378aef020e1f1b"}, + {file = "virtualenv-20.25.1-py3-none-any.whl", hash = "sha256:961c026ac520bac5f69acb8ea063e8a4f071bcc9457b9c1f28f6b085c511583a"}, + {file = "virtualenv-20.25.1.tar.gz", hash = "sha256:e08e13ecdca7a0bd53798f356d5831434afa5b07b93f0abdf0797b7a06ffe197"}, ] [package.dependencies] @@ -2619,4 +2705,4 @@ hdf5 = ["h5py"] [metadata] lock-version = "2.0" python-versions = "^3.7.1" -content-hash = "7f257d18b92faa0a8a5acc8bb04c3b80583a03265115c049c651b3f859da431a" +content-hash = "1572a843b2a77b5376349ff6cd212d39644931ab91b8e7b61c12709eda55617e" diff --git a/pyproject.toml b/pyproject.toml index 0aed7fb40..1d24ee7d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "octue" -version = "0.52.1" +version = "0.53.0" description = "A package providing template applications for data services, and a python SDK to the Octue API." readme = "README.md" authors = ["Marcus Lugg ", "Thomas Clark "] @@ -22,7 +22,7 @@ classifiers = [ [tool.poetry.dependencies] python = "^3.7.1" click = ">=7,<9" -coolname = "^1.1" +coolname = "^2" Flask = "^2" google-auth = ">=1.27.0,<3" google-cloud-pubsub = "^2.5" @@ -35,6 +35,8 @@ pyyaml = "^6" h5py = { version = "^3.6", optional = true } twined = "^0.5.1" packaging = ">=20.4" +google-cloud-bigquery = "^3.18.0" +db-dtypes = "^1.2.0" [tool.poetry.extras] hdf5 = ["h5py"] diff --git a/terraform/bigquery.tf b/terraform/bigquery.tf new file mode 100644 index 000000000..63abffe09 --- /dev/null +++ b/terraform/bigquery.tf @@ -0,0 +1,85 @@ +resource "google_bigquery_dataset" "test_dataset" { + dataset_id = "octue_sdk_python_test_dataset" + description = "A dataset for testing storing events for the Octue SDK." + location = "EU" + + labels = { + env = "default" + } +} + +resource "google_bigquery_table" "test_table" { + dataset_id = google_bigquery_dataset.test_dataset.dataset_id + table_id = "service-events" + clustering = ["sender", "question_uuid"] + + schema = <", ) - def test_ask_with_non_dictionary_message(self): - """Test that messages that aren't dictionaries fail validation.""" - messages = [ + def test_ask_with_non_dictionary_event(self): + """Test that events that aren't dictionaries fail validation.""" + events = [ ["hello"], ] - child_emulator = ChildEmulator(backend=self.BACKEND, messages=messages) + child_emulator = ChildEmulator(backend=self.BACKEND, events=events) with self.assertRaises(TypeError): child_emulator.ask(input_values={"hello": "world"}) - def test_ask_with_invalid_message_structure(self): - """Test that messages with an invalid structure fail validation.""" - messages = [ - {"message": "hello"}, + def test_ask_with_invalid_event_structure(self): + """Test that events with an invalid structure fail validation.""" + events = [ + {"event": "hello"}, ] - child_emulator = ChildEmulator(backend=self.BACKEND, messages=messages) + child_emulator = ChildEmulator(backend=self.BACKEND, events=events) with self.assertRaises(ValueError): child_emulator.ask(input_values={"hello": "world"}) - def test_ask_with_invalid_message_type(self): - """Test that messages with an invalid type fail validation.""" - messages = [ + def test_ask_with_invalid_event_type(self): + """Test that events with an invalid type fail validation.""" + events = [ {"kind": "hello", "content": [1, 2, 3, 4]}, ] - child_emulator = ChildEmulator(backend=self.BACKEND, messages=messages) + child_emulator = ChildEmulator(backend=self.BACKEND, events=events) with self.assertRaises(ValueError): child_emulator.ask(input_values={"hello": "world"}) @@ -58,23 +58,23 @@ def test_ask_with_invalid_message_type(self): # Re-enable this when schema validation has been sorted out in the child emulator. # def test_ask_with_invalid_result(self): # """Test that an invalid result fails validation.""" - # messages = [ + # events = [ # { # "kind": "result", # "wrong": "keys", # }, # ] # - # child_emulator = ChildEmulator(backend=self.BACKEND, messages=messages) + # child_emulator = ChildEmulator(backend=self.BACKEND, events=events) # # with self.assertRaises(ValueError): # child_emulator.ask(input_values={"hello": "world"}) - def test_ask_with_result_message(self): - """Test that result messages are returned by the emulator's ask method.""" + def test_ask_with_result_event(self): + """Test that result events are returned by the emulator's ask method.""" output_manifest = Manifest() - messages = [ + events = [ { "kind": "result", "output_values": [1, 2, 3, 4], @@ -82,7 +82,7 @@ def test_ask_with_result_message(self): }, ] - child_emulator = ChildEmulator(backend=self.BACKEND, messages=messages) + child_emulator = ChildEmulator(backend=self.BACKEND, events=events) result = child_emulator.ask(input_values={"hello": "world"}) self.assertEqual(result["output_values"], [1, 2, 3, 4]) @@ -92,7 +92,7 @@ def test_ask_with_input_manifest(self): """Test that a child emulator can accept an input manifest.""" input_manifest = Manifest() - messages = [ + events = [ { "kind": "result", "output_values": [1, 2, 3, 4], @@ -100,47 +100,47 @@ def test_ask_with_input_manifest(self): }, ] - child_emulator = ChildEmulator(backend=self.BACKEND, messages=messages) + child_emulator = ChildEmulator(backend=self.BACKEND, events=events) result = child_emulator.ask(input_values={"hello": "world"}, input_manifest=input_manifest) self.assertEqual(result["output_values"], [1, 2, 3, 4]) - def test_empty_output_returned_by_ask_if_no_result_present_in_messages(self): - """Test that an empty output is returned if no result message is present in the given messages.""" - child_emulator = ChildEmulator(backend=self.BACKEND, messages=[]) + def test_empty_output_returned_by_ask_if_no_result_present_in_events(self): + """Test that an empty output is returned if no result event is present in the given events.""" + child_emulator = ChildEmulator(backend=self.BACKEND, events=[]) result = child_emulator.ask(input_values={"hello": "world"}) self.assertEqual(result, {"output_values": None, "output_manifest": None}) def test_ask_with_log_record_with_missing_log_record_key(self): - """Test that an error is raised if a log record message missing the "log_record" key is given.""" - messages = [ + """Test that an error is raised if a log record event missing the "log_record" key is given.""" + events = [ { "kind": "log_record", } ] - child_emulator = ChildEmulator(backend=self.BACKEND, messages=messages) + child_emulator = ChildEmulator(backend=self.BACKEND, events=events) with self.assertRaises(ValueError): child_emulator.ask(input_values={"hello": "world"}) def test_ask_with_invalid_log_record(self): """Test that an invalid log record representation fails validation.""" - messages = [ + events = [ { "kind": "log_record", "log_record": [1, 2, 3], }, ] - child_emulator = ChildEmulator(backend=self.BACKEND, messages=messages) + child_emulator = ChildEmulator(backend=self.BACKEND, events=events) with self.assertRaises(TypeError): child_emulator.ask(input_values={"hello": "world"}) def test_ask_with_logs(self): """Test that log records can be handled by the emulator.""" - messages = [ + events = [ { "kind": "log_record", "log_record": {"msg": "Starting analysis.", "levelno": 20, "levelname": "INFO"}, @@ -151,7 +151,7 @@ def test_ask_with_logs(self): }, ] - child_emulator = ChildEmulator(backend=self.BACKEND, messages=messages) + child_emulator = ChildEmulator(backend=self.BACKEND, events=events) with self.assertLogs(level=logging.INFO) as logging_context: child_emulator.ask(input_values={"hello": "world"}) @@ -161,7 +161,7 @@ def test_ask_with_logs(self): def test_ask_with_logs_without_level_number_and_name(self): """Test that the 'INFO' log level is used if none is provided in the log record dictionaries.""" - messages = [ + events = [ { "kind": "log_record", "log_record": {"msg": "Starting analysis."}, @@ -172,7 +172,7 @@ def test_ask_with_logs_without_level_number_and_name(self): }, ] - child_emulator = ChildEmulator(backend=self.BACKEND, messages=messages) + child_emulator = ChildEmulator(backend=self.BACKEND, events=events) with self.assertLogs(level=logging.INFO) as logging_context: child_emulator.ask(input_values={"hello": "world"}) @@ -185,21 +185,21 @@ def test_ask_with_logs_without_level_number_and_name(self): def test_ask_with_invalid_exception(self): """Test that an invalid exception fails validation.""" - messages = [ + events = [ { "kind": "exception", "not": "an exception", }, ] - child_emulator = ChildEmulator(backend=self.BACKEND, messages=messages) + child_emulator = ChildEmulator(backend=self.BACKEND, events=events) with self.assertRaises(ValueError): child_emulator.ask(input_values={"hello": "world"}) def test_ask_with_exception(self): """Test that exceptions are raised by the emulator.""" - messages = [ + events = [ { "kind": "exception", "exception_type": "TypeError", @@ -207,7 +207,7 @@ def test_ask_with_exception(self): }, ] - child_emulator = ChildEmulator(backend=self.BACKEND, messages=messages) + child_emulator = ChildEmulator(backend=self.BACKEND, events=events) # Test that the exception was raised. with self.assertRaises(TypeError) as context: @@ -218,14 +218,14 @@ def test_ask_with_exception(self): def test_ask_with_monitor_message(self): """Test that monitor messages are handled by the emulator.""" - messages = [ + events = [ { "kind": "monitor_message", "data": "A sample monitor message.", }, ] - child_emulator = ChildEmulator(backend=self.BACKEND, messages=messages) + child_emulator = ChildEmulator(backend=self.BACKEND, events=events) monitor_messages = [] @@ -237,25 +237,25 @@ def test_ask_with_monitor_message(self): # Check that the monitor message handler has worked. self.assertEqual(monitor_messages, ["A sample monitor message."]) - def test_heartbeat_messages_are_ignored(self): - """Test that heartbeat messages are ignored by the emulator.""" - messages = [ + def test_heartbeat_events_are_ignored(self): + """Test that heartbeat events are ignored by the emulator.""" + events = [ { "kind": "heartbeat", "datetime": "2023-11-23T14:25:38.142884", }, ] - child_emulator = ChildEmulator(backend=self.BACKEND, messages=messages) + child_emulator = ChildEmulator(backend=self.BACKEND, events=events) with self.assertLogs(level=logging.WARNING) as logging_context: child_emulator.ask(input_values={"hello": "world"}) - self.assertIn("Heartbeat messages are ignored by the ChildEmulator.", logging_context.output[0]) + self.assertIn("Heartbeat events are ignored by the ChildEmulator.", logging_context.output[0]) - def test_messages_recorded_from_real_child_can_be_used_in_child_emulator(self): - """Test that messages recorded from a real child can be used as emulated messages in a child emulator (i.e. test - that the message format is unified between `Service` and `ChildEmulator`). + def test_events_recorded_from_real_child_can_be_used_in_child_emulator(self): + """Test that events recorded from a real child can be used as emulated events in a child emulator (i.e. test + that the event format is unified between `Service` and `ChildEmulator`). """ backend = GCPPubSubBackend(project_name="my-project") @@ -272,14 +272,14 @@ def error_run_function(*args, **kwargs): subscription, _ = parent.ask(service_id=child.id, input_values={}) parent.wait_for_answer(subscription=subscription) - child_emulator = ChildEmulator(messages=parent.received_messages) + child_emulator = ChildEmulator(events=parent.received_events) with self.assertRaises(OSError): child_emulator.ask(input_values={}) def test_ask_more_than_one_question(self): """Test than a child emulator can be asked more than one question without an error occurring.""" - messages = [ + events = [ { "kind": "result", "output_values": [1, 2, 3, 4], @@ -287,7 +287,7 @@ def test_ask_more_than_one_question(self): }, ] - child_emulator = ChildEmulator(backend=self.BACKEND, messages=messages) + child_emulator = ChildEmulator(backend=self.BACKEND, events=events) result_0 = child_emulator.ask(input_values={"hello": "world"}) result_1 = child_emulator.ask(input_values={"hello": "planet"}) self.assertEqual(result_0, result_1) @@ -305,11 +305,11 @@ def test_with_empty_file(self): result = child_emulator.ask(input_values={"hello": "world"}) self.assertEqual(result, {"output_values": None, "output_manifest": None}) - def test_with_only_messages(self): - """Test that a child emulator can be instantiated from a JSON file containing only the messages it should - produce, asked a question, and produce the expected log messages, monitor messages, and result. + def test_with_only_events(self): + """Test that a child emulator can be instantiated from a JSON file containing only the events it should + produce, asked a question, and produce the expected log events, monitor messages, and result. """ - emulator_file_path = os.path.join(self.TEST_FILES_DIRECTORY, "file_with_only_messages.json") + emulator_file_path = os.path.join(self.TEST_FILES_DIRECTORY, "file_with_only_events.json") child_emulator = ChildEmulator.from_file(emulator_file_path) self.assertEqual(child_emulator._child.backend.project_name, "emulated-project") @@ -333,7 +333,7 @@ def test_with_only_messages(self): def test_with_full_file(self): """Test that a child emulator can be instantiated from a JSON file containing all its instantiation arguments, - asked a question, and produce the correct messages. + asked a question, and produce the correct events. """ child_emulator = ChildEmulator.from_file(os.path.join(self.TEST_FILES_DIRECTORY, "full_file.json")) self.assertEqual(child_emulator.id, f"octue/my-child:{MOCK_SERVICE_REVISION_TAG}") @@ -359,8 +359,8 @@ def test_with_full_file(self): self.assertEqual(result, {"output_values": [1, 2, 3, 4, 5], "output_manifest": None}) def test_with_exception(self): - """Test that a child emulator can be instantiated from a JSON file including an exception in its messages and, - when asked a question, produce the messages prior to the exception before raising the exception. + """Test that a child emulator can be instantiated from a JSON file including an exception in its events and, + when asked a question, produce the events prior to the exception before raising the exception. """ child_emulator = ChildEmulator.from_file(os.path.join(self.TEST_FILES_DIRECTORY, "file_with_exception.json")) diff --git a/tests/cloud/emulators/valid_child_emulator_files/file_with_exception.json b/tests/cloud/emulators/valid_child_emulator_files/file_with_exception.json index dddcff096..cfe5a264b 100644 --- a/tests/cloud/emulators/valid_child_emulator_files/file_with_exception.json +++ b/tests/cloud/emulators/valid_child_emulator_files/file_with_exception.json @@ -1,5 +1,5 @@ { - "messages": [ + "events": [ { "kind": "log_record", "log_record": { "msg": "Starting analysis." } diff --git a/tests/cloud/emulators/valid_child_emulator_files/file_with_only_messages.json b/tests/cloud/emulators/valid_child_emulator_files/file_with_only_events.json similarity index 95% rename from tests/cloud/emulators/valid_child_emulator_files/file_with_only_messages.json rename to tests/cloud/emulators/valid_child_emulator_files/file_with_only_events.json index 1cacb36ff..0bbbc4a5c 100644 --- a/tests/cloud/emulators/valid_child_emulator_files/file_with_only_messages.json +++ b/tests/cloud/emulators/valid_child_emulator_files/file_with_only_events.json @@ -1,5 +1,5 @@ { - "messages": [ + "events": [ { "kind": "log_record", "log_record": { "msg": "Starting analysis." } diff --git a/tests/cloud/emulators/valid_child_emulator_files/file_with_output_manifest.json b/tests/cloud/emulators/valid_child_emulator_files/file_with_output_manifest.json index bd965a29f..596c193ae 100644 --- a/tests/cloud/emulators/valid_child_emulator_files/file_with_output_manifest.json +++ b/tests/cloud/emulators/valid_child_emulator_files/file_with_output_manifest.json @@ -1,5 +1,5 @@ { - "messages": [ + "events": [ { "kind": "result", "output_manifest": { diff --git a/tests/cloud/emulators/valid_child_emulator_files/full_file.json b/tests/cloud/emulators/valid_child_emulator_files/full_file.json index f16e56874..c3d0892b9 100644 --- a/tests/cloud/emulators/valid_child_emulator_files/full_file.json +++ b/tests/cloud/emulators/valid_child_emulator_files/full_file.json @@ -5,7 +5,7 @@ "project_name": "blah" }, "internal_service_name": "octue/my-service:2.3.0", - "messages": [ + "events": [ { "kind": "log_record", "log_record": { "msg": "Starting analysis." } diff --git a/tests/cloud/events/__init__.py b/tests/cloud/events/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/cloud/events/test_replayer.py b/tests/cloud/events/test_replayer.py new file mode 100644 index 000000000..99e073dd6 --- /dev/null +++ b/tests/cloud/events/test_replayer.py @@ -0,0 +1,81 @@ +import json +import logging +import os +import unittest + +from octue.cloud.events.replayer import EventReplayer +from tests import TEST_BUCKET_NAME, TESTS_DIR + + +class TestEventReplayer(unittest.TestCase): + def test_with_no_events(self): + """Test that `None` is returned if no events are passed in.""" + with self.assertLogs(level=logging.DEBUG) as logging_context: + result = EventReplayer().handle_events(events=[]) + + self.assertIsNone(result) + self.assertIn("No events (or no valid events) were received.", logging_context.output[0]) + + def test_with_no_valid_events(self): + """Test that `None` is returned if no valid events are received.""" + with self.assertLogs(level=logging.DEBUG) as logging_context: + result = EventReplayer().handle_events(events=[{"invalid": "event"}]) + + self.assertIsNone(result) + self.assertIn("received an event that doesn't conform", logging_context.output[1]) + self.assertIn("No events (or no valid events) were received.", logging_context.output[2]) + + def test_no_result_event(self): + """Test that `None` is returned if no result event is received.""" + event = { + "event": { + "datetime": "2024-03-06T15:44:18.156044", + "kind": "delivery_acknowledgement", + }, + "attributes": { + "datetime": "2024-04-11T10:46:48.236064", + "uuid": "a9de11b1-e88f-43fa-b3a4-40a590c3443f", + "order": "0", + "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", + "originator": "octue/test-service:1.0.0", + "sender": "octue/test-service:1.0.0", + "sender_type": "CHILD", + "sender_sdk_version": "0.51.0", + "recipient": "octue/another-service:3.2.1", + }, + } + + with self.assertLogs() as logging_context: + result = EventReplayer().handle_events(events=[event]) + + self.assertIsNone(result) + self.assertIn("question was delivered", logging_context.output[0]) + + def test_with_events_including_result_event(self): + """Test that stored events can be replayed and the outputs extracted from the "result" event.""" + with open(os.path.join(TESTS_DIR, "data", "events.json")) as f: + events = json.load(f) + + result = EventReplayer().handle_events(events) + self.assertEqual(result["output_values"], [1, 2, 3, 4, 5]) + + output_manifest = result["output_manifest"].to_primitive() + del output_manifest["datasets"]["example_dataset"]["id"] + + self.assertEqual( + output_manifest, + { + "id": "a13713ae-f207-41c6-9e29-0a848ced6039", + "name": None, + "datasets": { + "example_dataset": { + "name": "divergent-strange-gharial-of-pizza", + "tags": {}, + "labels": [], + "path": "https://storage.googleapis.com/octue-sdk-python-test-bucket/example_output_datasets" + "/example_dataset/.signed_metadata_files/divergent-strange-gharial-of-pizza", + "files": [f"gs://{TEST_BUCKET_NAME}/example_output_datasets/example_dataset/output.dat"], + } + }, + }, + ) diff --git a/tests/cloud/pub_sub/test_bigquery.py b/tests/cloud/pub_sub/test_bigquery.py new file mode 100644 index 000000000..0209e339a --- /dev/null +++ b/tests/cloud/pub_sub/test_bigquery.py @@ -0,0 +1,70 @@ +from unittest import TestCase +from unittest.mock import patch + +from octue.cloud.pub_sub.bigquery import get_events + + +class TestGetEvents(TestCase): + def test_error_raised_if_event_kind_invalid(self): + """Test that an error is raised if the event kind is invalid.""" + with self.assertRaises(ValueError): + get_events( + table_id="blah", + sender="octue/test-service:1.0.0", + question_uuid="blah", + kind="frisbee_tournament", + ) + + def test_without_kind(self): + """Test the query used to retrieve events of all kinds.""" + with patch("octue.cloud.pub_sub.bigquery.Client") as mock_client: + get_events(table_id="blah", sender="octue/test-service:1.0.0", question_uuid="blah") + + self.assertEqual( + mock_client.mock_calls[1].args[0], + "SELECT `event` FROM `blah`\nWHERE sender=@sender\nAND question_uuid=@question_uuid\n" + "ORDER BY `order`\nLIMIT @limit", + ) + + def test_with_kind(self): + """Test the query used to retrieve events of a specific kind.""" + with patch("octue.cloud.pub_sub.bigquery.Client") as mock_client: + get_events(table_id="blah", sender="octue/test-service:1.0.0", question_uuid="blah", kind="result") + + self.assertEqual( + mock_client.mock_calls[1].args[0], + "SELECT `event` FROM `blah`\nWHERE sender=@sender\nAND question_uuid=@question_uuid\n" + 'AND JSON_EXTRACT_SCALAR(event, "$.kind") = "result"\nORDER BY `order`\nLIMIT @limit', + ) + + def test_with_attributes(self): + """Test the query used to retrieve attributes in addition to events.""" + with patch("octue.cloud.pub_sub.bigquery.Client") as mock_client: + get_events( + table_id="blah", + sender="octue/test-service:1.0.0", + question_uuid="blah", + include_attributes=True, + ) + + self.assertEqual( + mock_client.mock_calls[1].args[0], + "SELECT `event`, `datetime`, `uuid`, `originator`, `sender`, `sender_type`, `sender_sdk_version`, `recipient`, `order`, `other_attributes` FROM `blah`\n" + "WHERE sender=@sender\nAND question_uuid=@question_uuid\nORDER BY `order`\nLIMIT @limit", + ) + + def test_with_backend_metadata(self): + """Test the query used to retrieve backend metadata in addition to events.""" + with patch("octue.cloud.pub_sub.bigquery.Client") as mock_client: + get_events( + table_id="blah", + sender="octue/test-service:1.0.0", + question_uuid="blah", + include_backend_metadata=True, + ) + + self.assertEqual( + mock_client.mock_calls[1].args[0], + "SELECT `event`, `backend`, `backend_metadata` FROM `blah`\n" + "WHERE sender=@sender\nAND question_uuid=@question_uuid\nORDER BY `order`\nLIMIT @limit", + ) diff --git a/tests/cloud/pub_sub/test_events.py b/tests/cloud/pub_sub/test_events.py new file mode 100644 index 000000000..e7b32ccb7 --- /dev/null +++ b/tests/cloud/pub_sub/test_events.py @@ -0,0 +1,709 @@ +import datetime +import math +import uuid +from unittest.mock import patch + +from octue.cloud.emulators._pub_sub import MESSAGES, MockMessage, MockService, MockSubscription +from octue.cloud.emulators.child import ServicePatcher +from octue.cloud.pub_sub.events import GoogleCloudPubSubEventHandler +from octue.resources.service_backends import GCPPubSubBackend +from tests import TEST_PROJECT_NAME +from tests.base import BaseTestCase + + +class TestGoogleCloudPubSubEventHandler(BaseTestCase): + service_patcher = ServicePatcher() + + @classmethod + def setUpClass(cls): + cls.service_patcher.start() + cls.question_uuid = str(uuid.uuid4()) + + cls.subscription = MockSubscription( + name=f"octue.services.my-org.my-service.1-0-0.answers.{cls.question_uuid}", + topic=cls.test_result_modifier.services_topic, + ) + cls.subscription.create() + + cls.parent = MockService( + service_id="my-org/my-service:1.0.0", + backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME), + ) + + @classmethod + def tearDownClass(cls): + """Stop the services patcher. + + :return None: + """ + cls.service_patcher.stop() + + def test_timeout(self): + """Test that a TimeoutError is raised if message handling takes longer than the given timeout.""" + event_handler = GoogleCloudPubSubEventHandler( + subscription=self.subscription, + recipient=self.parent, + event_handlers={"test": lambda message: None, "finish-test": lambda message: message}, + schema={}, + ) + + with self.assertRaises(TimeoutError): + event_handler.handle_events(timeout=0) + + def test_in_order_messages_are_handled_in_order(self): + """Test that messages received in order are handled in order.""" + event_handler = GoogleCloudPubSubEventHandler( + subscription=self.subscription, + recipient=self.parent, + event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, + schema={}, + ) + + child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) + + messages = [ + { + "event": {"kind": "test", "order": 0}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, + }, + { + "event": {"kind": "test", "order": 1}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, + }, + { + "event": {"kind": "test", "order": 2}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, + }, + { + "event": {"kind": "finish-test", "order": 3}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, + }, + ] + + for message in messages: + child._emit_event( + event=message["event"], + attributes=message["attributes"], + originator=self.parent.id, + recipient=self.parent.id, + order=message["event"]["order"], + ) + + result = event_handler.handle_events() + self.assertEqual(result, "This is the result.") + + self.assertEqual( + event_handler.handled_events, + [ + {"kind": "test", "order": 0}, + {"kind": "test", "order": 1}, + {"kind": "test", "order": 2}, + {"kind": "finish-test", "order": 3}, + ], + ) + + def test_out_of_order_messages_are_handled_in_order(self): + """Test that messages received out of order are handled in order.""" + event_handler = GoogleCloudPubSubEventHandler( + subscription=self.subscription, + recipient=self.parent, + event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, + schema={}, + ) + + child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) + + messages = [ + { + "event": {"kind": "test", "order": 1}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, + }, + { + "event": {"kind": "test", "order": 2}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, + }, + { + "event": {"kind": "test", "order": 0}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, + }, + { + "event": {"kind": "finish-test", "order": 3}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, + }, + ] + + for message in messages: + child._emit_event( + event=message["event"], + attributes=message["attributes"], + originator=self.parent.id, + recipient=self.parent.id, + order=message["event"]["order"], + ) + + result = event_handler.handle_events() + + self.assertEqual(result, "This is the result.") + + self.assertEqual( + event_handler.handled_events, + [ + {"kind": "test", "order": 0}, + {"kind": "test", "order": 1}, + {"kind": "test", "order": 2}, + {"kind": "finish-test", "order": 3}, + ], + ) + + def test_out_of_order_messages_with_end_message_first_are_handled_in_order(self): + """Test that messages received out of order and with the final message (the message that triggers a value to be + returned) are handled in order. + """ + event_handler = GoogleCloudPubSubEventHandler( + subscription=self.subscription, + recipient=self.parent, + event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, + schema={}, + ) + + child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) + + messages = [ + { + "event": {"kind": "finish-test", "order": 3}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, + }, + { + "event": {"kind": "test", "order": 1}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, + }, + { + "event": {"kind": "test", "order": 2}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, + }, + { + "event": {"kind": "test", "order": 0}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, + }, + ] + + for message in messages: + child._emit_event( + event=message["event"], + attributes=message["attributes"], + originator=self.parent.id, + recipient=self.parent.id, + order=message["event"]["order"], + ) + + result = event_handler.handle_events() + + self.assertEqual(result, "This is the result.") + + self.assertEqual( + event_handler.handled_events, + [ + {"kind": "test", "order": 0}, + {"kind": "test", "order": 1}, + {"kind": "test", "order": 2}, + {"kind": "finish-test", "order": 3}, + ], + ) + + def test_no_timeout(self): + """Test that message handling works with no timeout.""" + event_handler = GoogleCloudPubSubEventHandler( + subscription=self.subscription, + recipient=self.parent, + event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, + schema={}, + ) + + child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) + + messages = [ + { + "event": {"kind": "test", "order": 0}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, + }, + { + "event": {"kind": "test", "order": 1}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, + }, + { + "event": {"kind": "finish-test", "order": 2}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, + }, + ] + + for message in messages: + child._emit_event( + event=message["event"], + attributes=message["attributes"], + originator=self.parent.id, + recipient=self.parent.id, + order=message["event"]["order"], + ) + + result = event_handler.handle_events(timeout=None) + + self.assertEqual(result, "This is the result.") + self.assertEqual( + event_handler.handled_events, + [{"kind": "test", "order": 0}, {"kind": "test", "order": 1}, {"kind": "finish-test", "order": 2}], + ) + + def test_delivery_acknowledgement(self): + """Test that a delivery acknowledgement message is handled correctly.""" + event_handler = GoogleCloudPubSubEventHandler(subscription=self.subscription, recipient=self.parent) + child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) + + messages = [ + { + "event": { + "kind": "delivery_acknowledgement", + "datetime": datetime.datetime.utcnow().isoformat(), + "order": 0, + }, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, + }, + { + "event": {"kind": "result", "order": 1}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, + }, + ] + + for message in messages: + child._emit_event( + event=message["event"], + attributes=message["attributes"], + originator=self.parent.id, + recipient=self.parent.id, + order=message["event"]["order"], + ) + + result = event_handler.handle_events() + self.assertEqual(result, {"output_values": None, "output_manifest": None}) + + def test_error_raised_if_heartbeat_not_received_before_checked(self): + """Test that an error is raised if a heartbeat isn't received before a heartbeat is first checked for.""" + event_handler = GoogleCloudPubSubEventHandler(subscription=self.subscription, recipient=self.parent) + + with self.assertRaises(TimeoutError) as error: + event_handler.handle_events(maximum_heartbeat_interval=0) + + # Check that the timeout is due to a heartbeat not being received. + self.assertIn("heartbeat", error.exception.args[0]) + + def test_error_raised_if_heartbeats_stop_being_received(self): + """Test that an error is raised if heartbeats stop being received within the maximum interval.""" + event_handler = GoogleCloudPubSubEventHandler(subscription=self.subscription, recipient=self.parent) + event_handler._last_heartbeat = datetime.datetime.now() - datetime.timedelta(seconds=30) + + with self.assertRaises(TimeoutError) as error: + event_handler.handle_events(maximum_heartbeat_interval=0) + + self.assertIn("heartbeat", error.exception.args[0]) + + def test_error_not_raised_if_heartbeat_has_been_received_in_maximum_allowed_interval(self): + """Test that an error is not raised if a heartbeat has been received in the maximum allowed interval.""" + event_handler = GoogleCloudPubSubEventHandler(subscription=self.subscription, recipient=self.parent) + child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) + event_handler._last_heartbeat = datetime.datetime.now() + + messages = [ + { + "event": { + "kind": "delivery_acknowledgement", + "datetime": datetime.datetime.utcnow().isoformat(), + "order": 0, + }, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, + }, + { + "event": {"kind": "result", "order": 1}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, + }, + ] + + for message in messages: + child._emit_event( + event=message["event"], + attributes=message["attributes"], + originator=self.parent.id, + recipient=self.parent.id, + order=message["event"]["order"], + ) + + with patch( + "octue.cloud.pub_sub.events.GoogleCloudPubSubEventHandler._time_since_last_heartbeat", + datetime.timedelta(seconds=0), + ): + event_handler.handle_events(maximum_heartbeat_interval=0) + + def test_time_since_last_heartbeat_is_none_if_no_heartbeat_received_yet(self): + """Test that the time since the last heartbeat is `None` if no heartbeat has been received yet.""" + event_handler = GoogleCloudPubSubEventHandler(subscription=self.subscription, recipient=self.parent) + self.assertIsNone(event_handler._time_since_last_heartbeat) + + def test_total_run_time_is_none_if_handle_events_has_not_been_called(self): + """Test that the total run time for the message handler is `None` if the `handle_events` method has not been + called. + """ + event_handler = GoogleCloudPubSubEventHandler(subscription=self.subscription, recipient=self.parent) + self.assertIsNone(event_handler.total_run_time) + + def test_time_since_missing_message_is_none_if_no_unhandled_missing_messages(self): + """Test that the `time_since_missing_message` property is `None` if there are no unhandled missing messages.""" + event_handler = GoogleCloudPubSubEventHandler(subscription=self.subscription, recipient=self.parent) + self.assertIsNone(event_handler.time_since_missing_event) + + def test_missing_messages_at_start_can_be_skipped(self): + """Test that missing messages at the start of the event stream can be skipped if they aren't received after a + given time period if subsequent messages have been received. + """ + event_handler = GoogleCloudPubSubEventHandler( + subscription=self.subscription, + recipient=self.parent, + event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, + schema={}, + skip_missing_events_after=0, + ) + + child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) + + # Simulate the first two messages not being received. + messages = [ + { + "event": {"kind": "test", "order": 2}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, + }, + { + "event": {"kind": "test", "order": 3}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, + }, + { + "event": {"kind": "test", "order": 4}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, + }, + { + "event": {"kind": "finish-test", "order": 5}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, + }, + ] + + for message in messages: + child._emit_event( + event=message["event"], + attributes=message["attributes"], + originator=self.parent.id, + recipient=self.parent.id, + order=message["event"]["order"], + ) + + with self.assertLogs() as logging_context: + result = event_handler.handle_events() + + self.assertIn( + f"2 consecutive events missing for question {self.question_uuid!r} after 0s - skipping to next earliest " + f"waiting event (event 2).", + logging_context.output[0], + ) + + self.assertEqual(result, "This is the result.") + self.assertEqual( + event_handler.handled_events, + [ + {"kind": "test", "order": 2}, + {"kind": "test", "order": 3}, + {"kind": "test", "order": 4}, + {"kind": "finish-test", "order": 5}, + ], + ) + + def test_missing_messages_in_middle_can_skipped(self): + """Test that missing messages in the middle of the event stream can be skipped.""" + event_handler = GoogleCloudPubSubEventHandler( + subscription=self.subscription, + recipient=self.parent, + event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, + schema={}, + skip_missing_events_after=0, + ) + + child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) + + # Send three consecutive messages. + messages = [ + { + "event": {"kind": "test", "order": 0}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, + }, + { + "event": {"kind": "test", "order": 1}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, + }, + { + "event": {"kind": "test", "order": 2}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, + }, + ] + + for message in messages: + child._emit_event( + event=message["event"], + attributes=message["attributes"], + originator=self.parent.id, + recipient=self.parent.id, + order=message["event"]["order"], + ) + + # Send a final message. + child._emit_event( + event={"kind": "finish-test", "order": 5}, + attributes={"question_uuid": self.question_uuid, "sender_type": "CHILD"}, + originator=self.parent.id, + recipient=self.parent.id, + # Simulate missing messages. + order=5, + ) + + with self.assertLogs() as logging_context: + event_handler.handle_events() + + self.assertIn( + f"2 consecutive events missing for question {self.question_uuid!r} after 0s - skipping to next earliest " + f"waiting event (event 5).", + logging_context.output[0], + ) + + # Check that all the non-missing messages were handled. + self.assertEqual( + event_handler.handled_events, + [ + {"kind": "test", "order": 0}, + {"kind": "test", "order": 1}, + {"kind": "test", "order": 2}, + {"kind": "finish-test", "order": 5}, + ], + ) + + def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): + """Test that multiple blocks of missing messages in the middle of the event stream can be skipped.""" + event_handler = GoogleCloudPubSubEventHandler( + subscription=self.subscription, + recipient=self.parent, + event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, + schema={}, + skip_missing_events_after=0, + ) + + child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) + + # Send three consecutive messages. + messages = [ + { + "event": {"kind": "test", "order": 0}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, + }, + { + "event": {"kind": "test", "order": 1}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, + }, + { + "event": {"kind": "test", "order": 2}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, + }, + ] + + for message in messages: + child._emit_event( + event=message["event"], + attributes=message["attributes"], + originator=self.parent.id, + recipient=self.parent.id, + order=message["event"]["order"], + ) + + # Send another message. + child._emit_event( + event={"kind": "test", "order": 5}, + attributes={"order": 5, "question_uuid": self.question_uuid, "sender_type": "CHILD"}, + originator=self.parent.id, + recipient=self.parent.id, + # Simulate missing messages. + order=5, + ) + + # Send more consecutive messages. + messages = [ + { + "event": {"kind": "test", "order": 20}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, + }, + { + "event": {"kind": "test", "order": 21}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, + }, + { + "event": {"kind": "test", "order": 22}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, + }, + { + "event": {"kind": "finish-test", "order": 23}, + "attributes": {"question_uuid": self.question_uuid, "sender_type": "CHILD"}, + }, + ] + + for message in messages: + child._emit_event( + event=message["event"], + attributes=message["attributes"], + originator=self.parent.id, + recipient=self.parent.id, + # Simulate more missing messages. + order=message["event"]["order"], + ) + + with self.assertLogs() as logging_context: + event_handler.handle_events() + + self.assertIn( + f"2 consecutive events missing for question {self.question_uuid!r} after 0s - skipping to next earliest " + f"waiting event (event 5).", + logging_context.output[0], + ) + + self.assertIn( + f"14 consecutive events missing for question {self.question_uuid!r} after 0s - skipping to next earliest " + f"waiting event (event 20).", + logging_context.output[1], + ) + + # Check that all the non-missing messages were handled. + self.assertEqual( + event_handler.handled_events, + [ + {"kind": "test", "order": 0}, + {"kind": "test", "order": 1}, + {"kind": "test", "order": 2}, + {"kind": "test", "order": 5}, + {"kind": "test", "order": 20}, + {"kind": "test", "order": 21}, + {"kind": "test", "order": 22}, + {"kind": "finish-test", "order": 23}, + ], + ) + + def test_all_messages_missing_apart_from_result(self): + """Test that the result message is still handled if all other messages are missing.""" + event_handler = GoogleCloudPubSubEventHandler( + subscription=self.subscription, + recipient=self.parent, + event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, + schema={}, + skip_missing_events_after=0, + ) + + child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) + + # Send the result message. + child._emit_event( + event={"kind": "finish-test", "order": 1000}, + attributes={"question_uuid": self.question_uuid, "sender_type": "CHILD"}, + originator=self.parent.id, + recipient=self.parent.id, + # Simulate missing messages. + order=1000, + ) + + with self.assertLogs() as logging_context: + event_handler.handle_events() + + self.assertIn( + f"1000 consecutive events missing for question {self.question_uuid!r} after 0s - skipping to next earliest " + f"waiting event (event 1000).", + logging_context.output[0], + ) + # Check that the result message was handled. + self.assertEqual(event_handler.handled_events, [{"kind": "finish-test", "order": 1000}]) + + +class TestPullAndEnqueueAvailableMessages(BaseTestCase): + service_patcher = ServicePatcher() + + @classmethod + def setUpClass(cls): + cls.service_patcher.start() + cls.question_uuid = str(uuid.uuid4()) + + cls.subscription = MockSubscription( + name=f"my-org.my-service.1-0-0.answers.{cls.question_uuid}", + topic=cls.test_result_modifier.services_topic, + ) + cls.subscription.create() + + cls.parent = MockService( + service_id="my-org/my-service:1.0.0", + backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME), + ) + + @classmethod + def tearDownClass(cls): + """Stop the services patcher. + + :return None: + """ + cls.service_patcher.stop() + + def test_pull_and_enqueue_available_events(self): + """Test that pulling and enqueuing a message works.""" + event_handler = GoogleCloudPubSubEventHandler( + subscription=self.subscription, + recipient=self.parent, + event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, + schema={}, + ) + + event_handler.question_uuid = self.question_uuid + event_handler.child_sruid = "my-org/my-service:1.0.0" + event_handler.child_sdk_version = "0.1.3" + event_handler.waiting_events = {} + + # Enqueue a mock message for a mock subscription to receive. + mock_message = {"kind": "test"} + + MESSAGES[self.question_uuid] = [ + MockMessage.from_primitive( + mock_message, + attributes={ + "order": 0, + "question_uuid": self.question_uuid, + "originator": self.parent.id, + "sender": self.parent.id, + "sender_type": "CHILD", + "sender_sdk_version": "0.50.0", + "recipient": "my-org/my-service:1.0.0", + }, + ) + ] + + event_handler._pull_and_enqueue_available_events(timeout=10) + self.assertEqual(event_handler.waiting_events, {0: mock_message}) + + def test_timeout_error_raised_if_result_message_not_received_in_time(self): + """Test that a timeout error is raised if a result message is not received in time.""" + event_handler = GoogleCloudPubSubEventHandler( + subscription=self.subscription, + recipient=self.parent, + event_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, + ) + + event_handler.child_sdk_version = "0.1.3" + event_handler.waiting_events = {} + event_handler._start_time = 0 + + with self.assertRaises(TimeoutError): + event_handler._pull_and_enqueue_available_events(timeout=1e-6) + + self.assertEqual(event_handler._earliest_waiting_event_number, math.inf) diff --git a/tests/cloud/pub_sub/test_logging.py b/tests/cloud/pub_sub/test_logging.py index 44afbc933..77a8a76c2 100644 --- a/tests/cloud/pub_sub/test_logging.py +++ b/tests/cloud/pub_sub/test_logging.py @@ -3,8 +3,10 @@ from logging import makeLogRecord from unittest.mock import patch -from octue.cloud.emulators._pub_sub import SUBSCRIPTIONS, MockService, MockSubscription, MockTopic -from octue.cloud.pub_sub.logging import GooglePubSubHandler +from octue.cloud.emulators._pub_sub import MESSAGES, MockService +from octue.cloud.emulators.child import ServicePatcher +from octue.cloud.events.counter import EventCounter +from octue.cloud.pub_sub.logging import GoogleCloudPubSubHandler from octue.resources.service_backends import GCPPubSubBackend from tests.base import BaseTestCase @@ -14,29 +16,41 @@ def __repr__(self): return "NonJSONSerialisableInstance" -class TestGooglePubSubHandler(BaseTestCase): - def test_emit(self): - """Test the log message is published when `GooglePubSubHandler.emit` is called.""" - topic = MockTopic(name="world", project_name="blah") - topic.create() +class TestGoogleCloudPubSubHandler(BaseTestCase): + service_patcher = ServicePatcher() - question_uuid = "96d69278-44ac-4631-aeea-c90fb08a1b2b" - subscription = MockSubscription(name=f"world.answers.{question_uuid}", topic=topic, project_name="blah") - subscription.create() + @classmethod + def setUpClass(cls): + """Start the service patcher. - log_record = makeLogRecord({"msg": "Starting analysis."}) + :return None: + """ + cls.service_patcher.start() + + @classmethod + def tearDownClass(cls): + """Stop the services patcher. - backend = GCPPubSubBackend(project_name="blah") - service = MockService(backend=backend) + :return None: + """ + cls.service_patcher.stop() - GooglePubSubHandler( - message_sender=service._send_message, - topic=topic, + def test_emit(self): + """Test the log message is published when `GoogleCloudPubSubHandler.emit` is called.""" + question_uuid = "96d69278-44ac-4631-aeea-c90fb08a1b2b" + log_record = makeLogRecord({"msg": "Starting analysis."}) + service = MockService(backend=GCPPubSubBackend(project_name="blah")) + + GoogleCloudPubSubHandler( + event_emitter=service._emit_event, question_uuid=question_uuid, + originator="another/service:1.0.0", + recipient="another/service:1.0.0", + order=EventCounter(), ).emit(log_record) self.assertEqual( - json.loads(SUBSCRIPTIONS[subscription.name][0].data.decode())["log_record"]["msg"], + json.loads(MESSAGES[question_uuid][0].data.decode())["log_record"]["msg"], "Starting analysis.", ) @@ -44,9 +58,6 @@ def test_emit_with_non_json_serialisable_args(self): """Test that non-JSON-serialisable arguments to log messages are converted to their string representation before being serialised and published to the Pub/Sub topic. """ - topic = MockTopic(name="world-1", project_name="blah") - topic.create() - non_json_serialisable_thing = NonJSONSerialisable() # Check that it can't be serialised to JSON. @@ -57,14 +68,15 @@ def test_emit_with_non_json_serialisable_args(self): {"msg": "%r is not JSON-serialisable but can go into a log message", "args": (non_json_serialisable_thing,)} ) - backend = GCPPubSubBackend(project_name="blah") - service = MockService(backend=backend) + service = MockService(backend=GCPPubSubBackend(project_name="blah")) with patch("octue.cloud.emulators._pub_sub.MockPublisher.publish") as mock_publish: - GooglePubSubHandler( - message_sender=service._send_message, - topic=topic, + GoogleCloudPubSubHandler( + event_emitter=service._emit_event, question_uuid="question-uuid", + originator="another/service:1.0.0", + recipient="another/service:1.0.0", + order=EventCounter(), ).emit(record) self.assertEqual( diff --git a/tests/cloud/pub_sub/test_message_handler.py b/tests/cloud/pub_sub/test_message_handler.py deleted file mode 100644 index fe10a290f..000000000 --- a/tests/cloud/pub_sub/test_message_handler.py +++ /dev/null @@ -1,644 +0,0 @@ -import datetime -import math -import uuid -from unittest.mock import patch - -from octue.cloud.emulators._pub_sub import ( - SUBSCRIPTIONS, - MockMessage, - MockService, - MockSubscriber, - MockSubscription, - MockTopic, -) -from octue.cloud.emulators.child import ServicePatcher -from octue.cloud.pub_sub.message_handler import OrderedMessageHandler -from octue.resources.service_backends import GCPPubSubBackend -from tests import TEST_PROJECT_NAME -from tests.base import BaseTestCase - - -parent = MockService(service_id="my-org/my-service:1.0.0", backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) - - -def create_mock_topic_and_subscription(): - """Create a question UUID, mock topic, and mock subscription. - - :return (str, octue.cloud.emulators._pub_sub.MockTopic, octue.cloud.emulators._pub_sub.MockSubscription): question UUID, topic, and subscription - """ - question_uuid = str(uuid.uuid4()) - topic = MockTopic(name="my-org.my-service.1-0-0", project_name=TEST_PROJECT_NAME) - - subscription = MockSubscription( - name=f"my-org.my-service.1-0-0.answers.{question_uuid}", - topic=topic, - project_name=TEST_PROJECT_NAME, - ) - - subscription.create() - return question_uuid, topic, subscription - - -class TestOrderedMessageHandler(BaseTestCase): - def test_timeout(self): - """Test that a TimeoutError is raised if message handling takes longer than the given timeout.""" - question_uuid, _, mock_subscription = create_mock_topic_and_subscription() - - with patch("octue.cloud.pub_sub.message_handler.SubscriberClient", MockSubscriber): - message_handler = OrderedMessageHandler( - subscription=mock_subscription, - receiving_service=parent, - message_handlers={"test": lambda message: None, "finish-test": lambda message: message}, - schema={}, - ) - - with self.assertRaises(TimeoutError): - message_handler.handle_messages(timeout=0) - - def test_in_order_messages_are_handled_in_order(self): - """Test that messages received in order are handled in order.""" - question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() - - with patch("octue.cloud.pub_sub.message_handler.SubscriberClient", MockSubscriber): - message_handler = OrderedMessageHandler( - subscription=mock_subscription, - receiving_service=parent, - message_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, - schema={}, - ) - - child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) - - messages = [ - {"event": {"kind": "test"}, "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}}, - {"event": {"kind": "test"}, "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}}, - {"event": {"kind": "test"}, "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}}, - {"event": {"kind": "finish-test"}, "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}}, - ] - - for message in messages: - child._send_message(message=message["event"], attributes=message["attributes"], topic=mock_topic) - - result = message_handler.handle_messages() - self.assertEqual(result, "This is the result.") - - self.assertEqual( - message_handler.handled_messages, - [{"kind": "test"}, {"kind": "test"}, {"kind": "test"}, {"kind": "finish-test"}], - ) - - def test_out_of_order_messages_are_handled_in_order(self): - """Test that messages received out of order are handled in order.""" - question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() - - with patch("octue.cloud.pub_sub.message_handler.SubscriberClient", MockSubscriber): - message_handler = OrderedMessageHandler( - subscription=mock_subscription, - receiving_service=parent, - message_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, - schema={}, - ) - - child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) - - messages = [ - { - "event": {"kind": "test", "order": 1}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, - }, - { - "event": {"kind": "test", "order": 2}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, - }, - { - "event": {"kind": "test", "order": 0}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, - }, - { - "event": {"kind": "finish-test", "order": 3}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, - }, - ] - - for message in messages: - mock_topic.messages_published = message["event"]["order"] - child._send_message(message=message["event"], attributes=message["attributes"], topic=mock_topic) - - result = message_handler.handle_messages() - - self.assertEqual(result, "This is the result.") - - self.assertEqual( - message_handler.handled_messages, - [ - {"kind": "test", "order": 0}, - {"kind": "test", "order": 1}, - {"kind": "test", "order": 2}, - {"kind": "finish-test", "order": 3}, - ], - ) - - def test_out_of_order_messages_with_end_message_first_are_handled_in_order(self): - """Test that messages received out of order and with the final message (the message that triggers a value to be - returned) are handled in order. - """ - question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() - - with patch("octue.cloud.pub_sub.message_handler.SubscriberClient", MockSubscriber): - message_handler = OrderedMessageHandler( - subscription=mock_subscription, - receiving_service=parent, - message_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, - schema={}, - ) - - child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) - - messages = [ - { - "event": {"kind": "finish-test", "order": 3}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, - }, - { - "event": {"kind": "test", "order": 1}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, - }, - { - "event": {"kind": "test", "order": 2}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, - }, - { - "event": {"kind": "test", "order": 0}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, - }, - ] - - for message in messages: - mock_topic.messages_published = message["event"]["order"] - child._send_message(message=message["event"], attributes=message["attributes"], topic=mock_topic) - - result = message_handler.handle_messages() - - self.assertEqual(result, "This is the result.") - - self.assertEqual( - message_handler.handled_messages, - [ - {"kind": "test", "order": 0}, - {"kind": "test", "order": 1}, - {"kind": "test", "order": 2}, - {"kind": "finish-test", "order": 3}, - ], - ) - - def test_no_timeout(self): - """Test that message handling works with no timeout.""" - question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() - - with patch("octue.cloud.pub_sub.message_handler.SubscriberClient", MockSubscriber): - message_handler = OrderedMessageHandler( - subscription=mock_subscription, - receiving_service=parent, - message_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, - schema={}, - ) - - child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) - - messages = [ - { - "event": {"kind": "test", "order": 0}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, - }, - { - "event": {"kind": "test", "order": 1}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, - }, - { - "event": {"kind": "finish-test", "order": 2}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, - }, - ] - - for message in messages: - child._send_message(message=message["event"], attributes=message["attributes"], topic=mock_topic) - - result = message_handler.handle_messages(timeout=None) - - self.assertEqual(result, "This is the result.") - self.assertEqual( - message_handler.handled_messages, - [{"kind": "test", "order": 0}, {"kind": "test", "order": 1}, {"kind": "finish-test", "order": 2}], - ) - - def test_delivery_acknowledgement(self): - """Test that a delivery acknowledgement message is handled correctly.""" - question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() - - with patch("octue.cloud.pub_sub.message_handler.SubscriberClient", MockSubscriber): - message_handler = OrderedMessageHandler(subscription=mock_subscription, receiving_service=parent) - - child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) - - messages = [ - { - "event": {"kind": "delivery_acknowledgement", "datetime": datetime.datetime.utcnow().isoformat()}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, - }, - { - "event": {"kind": "result"}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, - }, - ] - - for message in messages: - child._send_message(message=message["event"], attributes=message["attributes"], topic=mock_topic) - - result = message_handler.handle_messages() - self.assertEqual(result, {"output_values": None, "output_manifest": None}) - - def test_error_raised_if_heartbeat_not_received_before_checked(self): - """Test that an error is raised if a heartbeat isn't received before a heartbeat is first checked for.""" - question_uuid, _, mock_subscription = create_mock_topic_and_subscription() - - with patch("octue.cloud.pub_sub.message_handler.SubscriberClient", MockSubscriber): - message_handler = OrderedMessageHandler(subscription=mock_subscription, receiving_service=parent) - - with self.assertRaises(TimeoutError) as error: - message_handler.handle_messages(maximum_heartbeat_interval=0) - - # Check that the timeout is due to a heartbeat not being received. - self.assertIn("heartbeat", error.exception.args[0]) - - def test_error_raised_if_heartbeats_stop_being_received(self): - """Test that an error is raised if heartbeats stop being received within the maximum interval.""" - question_uuid, _, mock_subscription = create_mock_topic_and_subscription() - - with patch("octue.cloud.pub_sub.message_handler.SubscriberClient", MockSubscriber): - message_handler = OrderedMessageHandler(subscription=mock_subscription, receiving_service=parent) - - message_handler._last_heartbeat = datetime.datetime.now() - datetime.timedelta(seconds=30) - - with self.assertRaises(TimeoutError) as error: - message_handler.handle_messages(maximum_heartbeat_interval=0) - - self.assertIn("heartbeat", error.exception.args[0]) - - def test_error_not_raised_if_heartbeat_has_been_received_in_maximum_allowed_interval(self): - """Test that an error is not raised if a heartbeat has been received in the maximum allowed interval.""" - question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() - - with patch("octue.cloud.pub_sub.message_handler.SubscriberClient", MockSubscriber): - message_handler = OrderedMessageHandler(subscription=mock_subscription, receiving_service=parent) - - message_handler._last_heartbeat = datetime.datetime.now() - - child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) - - messages = [ - { - "event": { - "kind": "delivery_acknowledgement", - "datetime": datetime.datetime.utcnow().isoformat(), - }, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, - }, - { - "event": {"kind": "result"}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, - }, - ] - - for message in messages: - child._send_message(message=message["event"], attributes=message["attributes"], topic=mock_topic) - - with patch( - "octue.cloud.pub_sub.message_handler.OrderedMessageHandler._time_since_last_heartbeat", - datetime.timedelta(seconds=0), - ): - message_handler.handle_messages(maximum_heartbeat_interval=0) - - def test_time_since_last_heartbeat_is_none_if_no_heartbeat_received_yet(self): - """Test that the time since the last heartbeat is `None` if no heartbeat has been received yet.""" - question_uuid, _, mock_subscription = create_mock_topic_and_subscription() - - with patch("octue.cloud.pub_sub.message_handler.SubscriberClient", MockSubscriber): - message_handler = OrderedMessageHandler(subscription=mock_subscription, receiving_service=parent) - - self.assertIsNone(message_handler._time_since_last_heartbeat) - - def test_total_run_time_is_none_if_handle_messages_has_not_been_called(self): - """Test that the total run time for the message handler is `None` if the `handle_messages` method has not been - called. - """ - question_uuid, _, mock_subscription = create_mock_topic_and_subscription() - message_handler = OrderedMessageHandler(subscription=mock_subscription, receiving_service=parent) - self.assertIsNone(message_handler.total_run_time) - - def test_time_since_missing_message_is_none_if_no_unhandled_missing_messages(self): - """Test that the `OrderedMessageHandler.time_since_missing_message` property is `None` if there are no unhandled - missing messages. - """ - question_uuid, _, mock_subscription = create_mock_topic_and_subscription() - message_handler = OrderedMessageHandler(subscription=mock_subscription, receiving_service=parent) - self.assertIsNone(message_handler.time_since_missing_message) - - def test_missing_messages_at_start_can_be_skipped(self): - """Test that missing messages at the start of the event stream can be skipped if they aren't received after a - given time period if subsequent messages have been received. - """ - question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() - - with patch("octue.cloud.pub_sub.message_handler.SubscriberClient", MockSubscriber): - message_handler = OrderedMessageHandler( - subscription=mock_subscription, - receiving_service=parent, - message_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, - schema={}, - skip_missing_messages_after=0, - ) - - # Simulate the first two messages not being received. - mock_topic.messages_published = 2 - - child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) - - messages = [ - { - "event": {"kind": "test", "order": 2}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, - }, - { - "event": {"kind": "test", "order": 3}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, - }, - { - "event": {"kind": "test", "order": 4}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, - }, - { - "event": {"kind": "finish-test", "order": 5}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, - }, - ] - - for message in messages: - child._send_message(message=message["event"], attributes=message["attributes"], topic=mock_topic) - - result = message_handler.handle_messages() - - self.assertEqual(result, "This is the result.") - self.assertEqual( - message_handler.handled_messages, - [ - {"kind": "test", "order": 2}, - {"kind": "test", "order": 3}, - {"kind": "test", "order": 4}, - {"kind": "finish-test", "order": 5}, - ], - ) - - def test_missing_messages_in_middle_can_skipped(self): - """Test that missing messages in the middle of the event stream can be skipped.""" - question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() - - with patch("octue.cloud.pub_sub.message_handler.SubscriberClient", MockSubscriber): - message_handler = OrderedMessageHandler( - subscription=mock_subscription, - receiving_service=parent, - message_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, - schema={}, - skip_missing_messages_after=0, - ) - - child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) - - # Send three consecutive messages. - messages = [ - { - "event": {"kind": "test", "order": 0}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, - }, - { - "event": {"kind": "test", "order": 1}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, - }, - { - "event": {"kind": "test", "order": 2}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, - }, - ] - - for message in messages: - child._send_message(message=message["event"], attributes=message["attributes"], topic=mock_topic) - - # Simulate missing messages. - mock_topic.messages_published = 5 - - # Send a final message. - child._send_message( - message={"kind": "finish-test", "order": 5}, - attributes={"question_uuid": question_uuid, "sender_type": "CHILD"}, - topic=mock_topic, - ) - - message_handler.handle_messages() - - # Check that all the non-missing messages were handled. - self.assertEqual( - message_handler.handled_messages, - [ - {"kind": "test", "order": 0}, - {"kind": "test", "order": 1}, - {"kind": "test", "order": 2}, - {"kind": "finish-test", "order": 5}, - ], - ) - - def test_multiple_blocks_of_missing_messages_in_middle_can_skipped(self): - """Test that multiple blocks of missing messages in the middle of the event stream can be skipped.""" - question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() - - with patch("octue.cloud.pub_sub.message_handler.SubscriberClient", MockSubscriber): - message_handler = OrderedMessageHandler( - subscription=mock_subscription, - receiving_service=parent, - message_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, - schema={}, - skip_missing_messages_after=0, - ) - - child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) - - # Send three consecutive messages. - messages = [ - { - "event": {"kind": "test", "order": 0}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, - }, - { - "event": {"kind": "test", "order": 1}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, - }, - { - "event": {"kind": "test", "order": 2}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, - }, - ] - - for message in messages: - child._send_message(message=message["event"], attributes=message["attributes"], topic=mock_topic) - - # Simulate missing messages. - mock_topic.messages_published = 5 - - # Send another message. - child._send_message( - message={"kind": "test", "order": 5}, - attributes={"message_number": 5, "question_uuid": question_uuid, "sender_type": "CHILD"}, - topic=mock_topic, - ) - - # Simulate more missing messages. - mock_topic.messages_published = 20 - - # Send more consecutive messages. - messages = [ - { - "event": {"kind": "test", "order": 20}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, - }, - { - "event": {"kind": "test", "order": 21}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, - }, - { - "event": {"kind": "test", "order": 22}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, - }, - { - "event": {"kind": "finish-test", "order": 23}, - "attributes": {"question_uuid": question_uuid, "sender_type": "CHILD"}, - }, - ] - - for message in messages: - child._send_message(message=message["event"], attributes=message["attributes"], topic=mock_topic) - - message_handler.handle_messages() - - # Check that all the non-missing messages were handled. - self.assertEqual( - message_handler.handled_messages, - [ - {"kind": "test", "order": 0}, - {"kind": "test", "order": 1}, - {"kind": "test", "order": 2}, - {"kind": "test", "order": 5}, - {"kind": "test", "order": 20}, - {"kind": "test", "order": 21}, - {"kind": "test", "order": 22}, - {"kind": "finish-test", "order": 23}, - ], - ) - - def test_all_messages_missing_apart_from_result(self): - """Test that the result message is still handled if all other messages are missing.""" - question_uuid, mock_topic, mock_subscription = create_mock_topic_and_subscription() - - with patch("octue.cloud.pub_sub.message_handler.SubscriberClient", MockSubscriber): - message_handler = OrderedMessageHandler( - subscription=mock_subscription, - receiving_service=parent, - message_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, - schema={}, - skip_missing_messages_after=0, - ) - - child = MockService(backend=GCPPubSubBackend(project_name=TEST_PROJECT_NAME)) - - # Simulate missing messages. - mock_topic.messages_published = 1000 - - # Send the result message. - child._send_message( - message={"kind": "finish-test", "order": 1000}, - attributes={"question_uuid": question_uuid, "sender_type": "CHILD"}, - topic=mock_topic, - ) - - message_handler.handle_messages() - - # Check that the result message was handled. - self.assertEqual(message_handler.handled_messages, [{"kind": "finish-test", "order": 1000}]) - - -class TestPullAndEnqueueAvailableMessages(BaseTestCase): - def test_pull_and_enqueue_available_messages(self): - """Test that pulling and enqueuing a message works.""" - question_uuid, mock_topic, _ = create_mock_topic_and_subscription() - - with ServicePatcher(): - mock_subscription = MockSubscription( - name=f"my-org.my-service.1-0-0.answers.{question_uuid}", - topic=mock_topic, - project_name=TEST_PROJECT_NAME, - ) - - message_handler = OrderedMessageHandler( - subscription=mock_subscription, - receiving_service=parent, - message_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, - schema={}, - ) - - message_handler._child_sdk_version = "0.1.3" - message_handler.waiting_messages = {} - - # Enqueue a mock message for a mock subscription to receive. - mock_message = {"kind": "test"} - - SUBSCRIPTIONS[mock_subscription.name] = [ - MockMessage.from_primitive( - mock_message, - attributes={ - "sender_type": "CHILD", - "message_number": 0, - "question_uuid": question_uuid, - "version": "0.50.0", - }, - ) - ] - - message_handler._pull_and_enqueue_available_messages(timeout=10) - self.assertEqual(message_handler.waiting_messages, {0: mock_message}) - self.assertEqual(message_handler._earliest_waiting_message_number, 0) - - def test_timeout_error_raised_if_result_message_not_received_in_time(self): - """Test that a timeout error is raised if a result message is not received in time.""" - question_uuid, mock_topic, _ = create_mock_topic_and_subscription() - - with ServicePatcher(): - mock_subscription = MockSubscription( - name=f"my-org.my-service.1-0-0.answers.{question_uuid}", - topic=mock_topic, - project_name=TEST_PROJECT_NAME, - ) - - message_handler = OrderedMessageHandler( - subscription=mock_subscription, - receiving_service=parent, - message_handlers={"test": lambda message: None, "finish-test": lambda message: "This is the result."}, - ) - - message_handler._child_sdk_version = "0.1.3" - message_handler.waiting_messages = {} - message_handler._start_time = 0 - - # Create a mock subscription. - SUBSCRIPTIONS[mock_subscription.name] = [] - - with self.assertRaises(TimeoutError): - message_handler._pull_and_enqueue_available_messages(timeout=1e-6) - - self.assertEqual(message_handler._earliest_waiting_message_number, math.inf) diff --git a/tests/cloud/pub_sub/test_service.py b/tests/cloud/pub_sub/test_service.py index 9ae63cbfb..8e6526acb 100644 --- a/tests/cloud/pub_sub/test_service.py +++ b/tests/cloud/pub_sub/test_service.py @@ -15,7 +15,6 @@ DifferentMockAnalysis, MockAnalysis, MockAnalysisWithOutputManifest, - MockPullResponse, MockService, MockSubscription, MockTopic, @@ -43,6 +42,22 @@ class TestService(BaseTestCase): service_patcher = ServicePatcher() + @classmethod + def setUpClass(cls): + """Start the service patcher. + + :return None: + """ + cls.service_patcher.start() + + @classmethod + def tearDownClass(cls): + """Stop the services patcher. + + :return None: + """ + cls.service_patcher.stop() + def test_repr(self): """Test that services are represented as a string correctly.""" service = Service(backend=BACKEND) @@ -66,26 +81,21 @@ def test_service_id_cannot_be_non_none_empty_value(self): def test_serve_fails_if_service_with_same_id_already_exists(self): """Test that serving a service fails if a service with the same name already exists.""" - with self.service_patcher: - with patch( - "octue.cloud.pub_sub.service.Topic.create", - side_effect=google.api_core.exceptions.AlreadyExists(""), - ): - with self.assertRaises(exceptions.ServiceAlreadyExists): - MockService( - backend=BACKEND, - service_id=f"my-org/existing-service:{MOCK_SERVICE_REVISION_TAG}", - ).serve() + with patch( + "octue.cloud.pub_sub.service.Subscription.create", + side_effect=google.api_core.exceptions.AlreadyExists(""), + ): + with self.assertRaises(exceptions.ServiceAlreadyExists): + MockService(backend=BACKEND, service_id=f"my-org/existing-service:{MOCK_SERVICE_REVISION_TAG}").serve() def test_serve(self): """Test that serving works with a unique service ID. Test that the returned future has itself been returned and that the returned subscriber has been closed. """ - with self.service_patcher: - future, subscriber = MockService( - backend=BACKEND, - service_id=f"my-org/existing-service:{MOCK_SERVICE_REVISION_TAG}", - ).serve() + future, subscriber = MockService( + backend=BACKEND, + service_id=f"my-org/existing-service:{MOCK_SERVICE_REVISION_TAG}", + ).serve() self.assertFalse(future.cancelled) self.assertTrue(future.returned) @@ -95,107 +105,91 @@ def test_serve_detached(self): """Test that, when serving a service in detached mode, the returned future is not cancelled or returned and that the returned subscriber is not closed. """ - with self.service_patcher: - service = MockService(backend=BACKEND, service_id=f"my-org/existing-service-d:{MOCK_SERVICE_REVISION_TAG}") - future, subscriber = service.serve(detach=True) + service = MockService(backend=BACKEND, service_id=f"my-org/existing-service-d:{MOCK_SERVICE_REVISION_TAG}") + future, subscriber = service.serve(detach=True) self.assertFalse(future.cancelled) self.assertFalse(future.returned) self.assertFalse(subscriber.closed) - def test_ask_on_non_existent_service_results_in_error(self): - """Test that trying to ask a question to a non-existent service (i.e. one without a topic in Google Pub/Sub) - results in an error. - """ - with patch("octue.cloud.pub_sub.service.Topic", new=MockTopic): - service = MockService(backend=BACKEND) + def test_missing_services_topic_results_in_error(self): + """Test that an error is raised if the services topic doesn't exist.""" + service = MockService(backend=BACKEND) + with patch("octue.cloud.emulators._pub_sub.TOPICS", set()): with self.assertRaises(exceptions.ServiceNotFound): - service.ask( - service_id=f"my-org/existing-service:{MOCK_SERVICE_REVISION_TAG}", - input_values=[1, 2, 3, 4], - ) + service.services_topic def test_ask_unregistered_service_revision_when_service_registries_specified_results_in_error(self): """Test that an error is raised if attempting to ask an unregistered service a question when service registries are being used. """ - with patch("octue.cloud.pub_sub.service.Topic", new=MockTopic): - service = MockService( - backend=BACKEND, - service_registries=[{"name": "Octue Registry", "endpoint": "https://blah.com/services"}], - ) + service = MockService( + backend=BACKEND, + service_registries=[{"name": "Octue Registry", "endpoint": "https://blah.com/services"}], + ) - mock_response = requests.Response() - mock_response.status_code = 404 + mock_response = requests.Response() + mock_response.status_code = 404 - with patch("requests.get", return_value=mock_response): - with self.assertRaises(exceptions.ServiceNotFound): - service.ask( - service_id=f"my-org/unregistered-service:{MOCK_SERVICE_REVISION_TAG}", - input_values=[1, 2, 3, 4], - ) + with patch("requests.get", return_value=mock_response): + with self.assertRaises(exceptions.ServiceNotFound): + service.ask( + service_id=f"my-org/unregistered-service:{MOCK_SERVICE_REVISION_TAG}", + input_values=[1, 2, 3, 4], + ) def test_ask_unregistered_service_with_no_revision_tag_when_service_registries_specified_results_in_error(self): """Test that an error is raised when attempting to ask a question to an unregistered service without including revision tag when service registries are being used. """ - with patch("octue.cloud.pub_sub.service.Topic", new=MockTopic): - service = MockService( - backend=BACKEND, - service_registries=[{"name": "Octue Registry", "endpoint": "https://blah.com/services"}], - ) + service = MockService( + backend=BACKEND, + service_registries=[{"name": "Octue Registry", "endpoint": "https://blah.com/services"}], + ) - mock_response = requests.Response() - mock_response.status_code = 404 + mock_response = requests.Response() + mock_response.status_code = 404 - with patch("requests.get", return_value=mock_response): - with self.assertRaises(exceptions.ServiceNotFound): - service.ask(service_id="my-org/unregistered-service", input_values=[1, 2, 3, 4]) + with patch("requests.get", return_value=mock_response): + with self.assertRaises(exceptions.ServiceNotFound): + service.ask(service_id="my-org/unregistered-service", input_values=[1, 2, 3, 4]) def test_ask_service_with_no_revision_tag_when_service_registries_not_specified_results_in_error(self): """Test that an error is raised when attempting to ask a question to a service without including a revision tag when service registries are not being used. """ - with patch("octue.cloud.pub_sub.service.Topic", new=MockTopic): - service = MockService(backend=BACKEND) + service = MockService(backend=BACKEND) - mock_response = requests.Response() - mock_response.status_code = 404 + mock_response = requests.Response() + mock_response.status_code = 404 - with patch("requests.get", return_value=mock_response): - with self.assertRaises(exceptions.InvalidServiceID): - service.ask(service_id="my-org/unregistered-service", input_values=[1, 2, 3, 4]) + with patch("requests.get", return_value=mock_response): + with self.assertRaises(exceptions.InvalidServiceID): + service.ask(service_id="my-org/unregistered-service", input_values=[1, 2, 3, 4]) def test_timeout_error_raised_if_no_messages_received_when_waiting(self): """Test that a TimeoutError is raised if no messages are received while waiting.""" mock_topic = MockTopic(name="amazing.service.9-9-9", project_name=TEST_PROJECT_NAME) - - mock_subscription = MockSubscription( - name="amazing.service.9-9-9", - topic=mock_topic, - project_name=TEST_PROJECT_NAME, - ) + mock_subscription = MockSubscription(name="amazing.service.9-9-9", topic=mock_topic) service = Service(backend=BACKEND) - with patch("octue.cloud.pub_sub.service.pubsub_v1.SubscriberClient.pull", return_value=MockPullResponse()): - with self.assertRaises(TimeoutError): - service.wait_for_answer(subscription=mock_subscription, timeout=0.01) + with self.assertRaises(TimeoutError): + service.wait_for_answer(subscription=mock_subscription, timeout=0.01) def test_error_raised_if_attempting_to_wait_for_answer_from_push_subscription(self): """Test that an error is raised if attempting to wait for an answer from a push subscription.""" - mock_subscription = MockSubscription( + service = Service(backend=BACKEND) + + subscription = MockSubscription( name="world", topic=MockTopic(name="world", project_name=TEST_PROJECT_NAME), - project_name=TEST_PROJECT_NAME, push_endpoint="https://example.com/endpoint", ) - service = Service(backend=BACKEND) - - with self.assertRaises(exceptions.PushSubscriptionCannotBePulled): - service.wait_for_answer(subscription=mock_subscription) + with self.assertRaises(exceptions.NotAPullSubscription): + service.wait_for_answer(subscription=subscription) def test_exceptions_in_responder_are_handled_and_sent_to_asker(self): """Test that exceptions raised in the child service are handled and sent back to the asker.""" @@ -205,12 +199,11 @@ def test_exceptions_in_responder_are_handled_and_sent_to_asker(self): parent = MockService(backend=BACKEND, children={child.id: child}) - with self.service_patcher: - child.serve() - subscription, _ = parent.ask(service_id=child.id, input_values={}) + child.serve() + subscription, _ = parent.ask(service_id=child.id, input_values={}) - with self.assertRaises(twined.exceptions.InvalidManifestContents) as context: - parent.wait_for_answer(subscription=subscription) + with self.assertRaises(twined.exceptions.InvalidManifestContents) as context: + parent.wait_for_answer(subscription=subscription) self.assertIn("'met_mast_id' is a required property", context.exception.args[0]) @@ -221,12 +214,11 @@ def test_exceptions_with_multiple_arguments_in_responder_are_handled_and_sent_to child = self.make_new_child_with_error(FileNotFoundError(2, "No such file or directory: 'blah'")) parent = MockService(backend=BACKEND, children={child.id: child}) - with self.service_patcher: - child.serve() - subscription, _ = parent.ask(service_id=child.id, input_values={}) + child.serve() + subscription, _ = parent.ask(service_id=child.id, input_values={}) - with self.assertRaises(FileNotFoundError) as context: - parent.wait_for_answer(subscription) + with self.assertRaises(FileNotFoundError) as context: + parent.wait_for_answer(subscription) self.assertIn("[Errno 2] No such file or directory: 'blah'", format(context.exception)) @@ -239,12 +231,11 @@ class AnUnknownException(Exception): child = self.make_new_child_with_error(AnUnknownException("This is an exception unknown to the asker.")) parent = MockService(backend=BACKEND, children={child.id: child}) - with self.service_patcher: - child.serve() - subscription, _ = parent.ask(service_id=child.id, input_values={}) + child.serve() + subscription, _ = parent.ask(service_id=child.id, input_values={}) - with self.assertRaises(Exception) as context: - parent.wait_for_answer(subscription) + with self.assertRaises(Exception) as context: + parent.wait_for_answer(subscription) self.assertEqual(type(context.exception).__name__, "AnUnknownException") self.assertIn("This is an exception unknown to the asker.", context.exception.args[0]) @@ -258,10 +249,9 @@ def test_ask_with_real_run_function_with_no_log_message_forwarding(self): parent = MockService(backend=BACKEND, children={child.id: child}) with self.assertLogs() as logging_context: - with self.service_patcher: - child.serve() - subscription, _ = parent.ask(service_id=child.id, input_values={}, subscribe_to_logs=False) - answer = parent.wait_for_answer(subscription) + child.serve() + subscription, _ = parent.ask(service_id=child.id, input_values={}, subscribe_to_logs=False) + answer = parent.wait_for_answer(subscription) self.assertEqual( answer, @@ -284,10 +274,9 @@ def test_ask_with_real_run_function_with_log_message_forwarding(self): parent = MockService(backend=BACKEND, children={child.id: child}) with self.assertLogs() as logs_context_manager: - with self.service_patcher: - child.serve() - subscription, _ = parent.ask(service_id=child.id, input_values={}, subscribe_to_logs=True) - answer = parent.wait_for_answer(subscription) + child.serve() + subscription, _ = parent.ask(service_id=child.id, input_values={}, subscribe_to_logs=True) + answer = parent.wait_for_answer(subscription) self.assertEqual( answer, @@ -330,10 +319,9 @@ def mock_app(analysis): parent = MockService(backend=BACKEND, children={child.id: child}) with self.assertLogs(level=logging.ERROR) as logs_context_manager: - with self.service_patcher: - child.serve() - subscription, _ = parent.ask(service_id=child.id, input_values={}, subscribe_to_logs=True) - parent.wait_for_answer(subscription) + child.serve() + subscription, _ = parent.ask(service_id=child.id, input_values={}, subscribe_to_logs=True) + parent.wait_for_answer(subscription) error_logged = False @@ -372,12 +360,11 @@ def mock_app(analysis): child = MockService(backend=BACKEND, run_function=create_run_function_with_monitoring()) parent = MockService(backend=BACKEND, children={child.id: child}) - with self.service_patcher: - child.serve() - subscription, _ = parent.ask(child.id, input_values={}) + child.serve() + subscription, _ = parent.ask(child.id, input_values={}) - monitoring_data = [] - parent.wait_for_answer(subscription, handle_monitor_message=lambda data: monitoring_data.append(data)) + monitoring_data = [] + parent.wait_for_answer(subscription, handle_monitor_message=lambda data: monitoring_data.append(data)) self.assertEqual( monitoring_data, @@ -411,16 +398,12 @@ def mock_app(analysis): child = MockService(backend=BACKEND, run_function=create_run_function_with_monitoring()) parent = MockService(backend=BACKEND, children={child.id: child}) - with self.service_patcher: - child.serve() - subscription, _ = parent.ask(child.id, input_values={}) - monitoring_data = [] + child.serve() + subscription, _ = parent.ask(child.id, input_values={}) + monitoring_data = [] - with self.assertRaises(InvalidMonitorMessage): - parent.wait_for_answer( - subscription, - handle_monitor_message=lambda data: monitoring_data.append(data), - ) + with self.assertRaises(InvalidMonitorMessage): + parent.wait_for_answer(subscription, handle_monitor_message=lambda data: monitoring_data.append(data)) self.assertEqual(monitoring_data, [{"status": "my first monitor message"}]) @@ -428,36 +411,29 @@ def test_ask_with_non_json_python_primitive_input_values(self): """Test that non-JSON python primitive values (in this case a set and a datetime) can be sent and received by services. """ + input_values = {"my_set": {1, 2, 3}, "my_datetime": datetime.datetime.now()} def run_function(analysis_id, input_values, *args, **kwargs): return MockAnalysis(output_values=input_values) child = MockService(backend=BACKEND, run_function=lambda *args, **kwargs: run_function(*args, **kwargs)) parent = MockService(backend=BACKEND, children={child.id: child}) + child.serve() - input_values = {"my_set": {1, 2, 3}, "my_datetime": datetime.datetime.now()} - - with self.service_patcher: - child.serve() - - subscription, _ = parent.ask( - service_id=child.id, - input_values=input_values, - subscribe_to_logs=True, - save_diagnostics="SAVE_DIAGNOSTICS_ON_CRASH", - ) - - answer = parent.wait_for_answer(subscription) + subscription, _ = parent.ask( + service_id=child.id, + input_values=input_values, + subscribe_to_logs=True, + save_diagnostics="SAVE_DIAGNOSTICS_ON_CRASH", + ) + answer = parent.wait_for_answer(subscription) self.assertEqual(answer["output_values"], input_values) def test_ask_with_input_manifest(self): """Test that a service can ask a question including an input manifest to another service that is serving and receive an answer. """ - child = self.make_new_child(BACKEND, run_function_returnee=MockAnalysis()) - parent = MockService(backend=BACKEND, children={child.id: child}) - dataset_path = f"gs://{TEST_BUCKET_NAME}/my-dataset" input_manifest = Manifest( @@ -469,12 +445,13 @@ def test_ask_with_input_manifest(self): } ) - with self.service_patcher: - child.serve() + child = self.make_new_child(BACKEND, run_function_returnee=MockAnalysis()) + parent = MockService(backend=BACKEND, children={child.id: child}) + child.serve() - with patch("google.cloud.storage.blob.Blob.generate_signed_url", new=mock_generate_signed_url): - subscription, _ = parent.ask(service_id=child.id, input_values={}, input_manifest=input_manifest) - answer = parent.wait_for_answer(subscription) + with patch("google.cloud.storage.blob.Blob.generate_signed_url", new=mock_generate_signed_url): + subscription, _ = parent.ask(service_id=child.id, input_values={}, input_manifest=input_manifest) + answer = parent.wait_for_answer(subscription) self.assertEqual( answer, @@ -485,9 +462,6 @@ def test_ask_with_input_manifest_and_no_input_values(self): """Test that a service can ask a question including an input manifest and no input values to another service that is serving and receive an answer. """ - child = self.make_new_child(BACKEND, run_function_returnee=MockAnalysis()) - parent = MockService(backend=BACKEND, children={child.id: child}) - dataset_path = f"gs://{TEST_BUCKET_NAME}/my-dataset" input_manifest = Manifest( @@ -499,12 +473,13 @@ def test_ask_with_input_manifest_and_no_input_values(self): } ) - with self.service_patcher: - child.serve() + child = self.make_new_child(BACKEND, run_function_returnee=MockAnalysis()) + parent = MockService(backend=BACKEND, children={child.id: child}) + child.serve() - with patch("google.cloud.storage.blob.Blob.generate_signed_url", new=mock_generate_signed_url): - subscription, _ = parent.ask(service_id=child.id, input_manifest=input_manifest) - answer = parent.wait_for_answer(subscription) + with patch("google.cloud.storage.blob.Blob.generate_signed_url", new=mock_generate_signed_url): + subscription, _ = parent.ask(service_id=child.id, input_manifest=input_manifest) + answer = parent.wait_for_answer(subscription) self.assertEqual( answer, @@ -545,18 +520,16 @@ def run_function(*args, **kwargs): child = MockService(backend=BACKEND, run_function=run_function) parent = MockService(backend=BACKEND, children={child.id: child}) + child.serve() - with self.service_patcher: - child.serve() - - subscription, _ = parent.ask( - service_id=child.id, - input_values={}, - input_manifest=manifest, - allow_local_files=True, - ) + subscription, _ = parent.ask( + service_id=child.id, + input_values={}, + input_manifest=manifest, + allow_local_files=True, + ) - answer = parent.wait_for_answer(subscription) + answer = parent.wait_for_answer(subscription) self.assertEqual(answer["output_values"], "This is a local file.") @@ -564,11 +537,10 @@ def test_ask_with_output_manifest(self): """Test that a service can receive an output manifest as part of the answer to a question.""" child = self.make_new_child(BACKEND, run_function_returnee=MockAnalysisWithOutputManifest()) parent = MockService(backend=BACKEND, children={child.id: child}) + child.serve() - with self.service_patcher: - child.serve() - subscription, _ = parent.ask(service_id=child.id, input_values={}) - answer = parent.wait_for_answer(subscription) + subscription, _ = parent.ask(service_id=child.id, input_values={}) + answer = parent.wait_for_answer(subscription) self.assertEqual(answer["output_values"], MockAnalysisWithOutputManifest.output_values) self.assertEqual(answer["output_manifest"].id, MockAnalysisWithOutputManifest.output_manifest.id) @@ -577,14 +549,12 @@ def test_service_can_ask_multiple_questions_to_child(self): """Test that a service can ask multiple questions to the same child and expect replies to them all.""" child = self.make_new_child(BACKEND, run_function_returnee=MockAnalysis()) parent = MockService(backend=BACKEND, children={child.id: child}) + child.serve() + answers = [] - with self.service_patcher: - child.serve() - answers = [] - - for i in range(5): - subscription, _ = parent.ask(service_id=child.id, input_values={}) - answers.append(parent.wait_for_answer(subscription)) + for i in range(5): + subscription, _ = parent.ask(service_id=child.id, input_values={}) + answers.append(parent.wait_for_answer(subscription)) for answer in answers: self.assertEqual( @@ -596,18 +566,16 @@ def test_service_can_ask_questions_to_multiple_children(self): """Test that a service can ask questions to different children and expect replies to them all.""" child_1 = self.make_new_child(BACKEND, run_function_returnee=MockAnalysis()) child_2 = self.make_new_child(BACKEND, run_function_returnee=DifferentMockAnalysis()) - parent = MockService(backend=BACKEND, children={child_1.id: child_1, child_2.id: child_2}) - with self.service_patcher: - child_1.serve() - child_2.serve() + child_1.serve() + child_2.serve() - subscription, _ = parent.ask(service_id=child_1.id, input_values={}) - answer_1 = parent.wait_for_answer(subscription) + subscription, _ = parent.ask(service_id=child_1.id, input_values={}) + answer_1 = parent.wait_for_answer(subscription) - subscription, _ = parent.ask(service_id=child_2.id, input_values={}) - answer_2 = parent.wait_for_answer(subscription) + subscription, _ = parent.ask(service_id=child_2.id, input_values={}) + answer_2 = parent.wait_for_answer(subscription) self.assertEqual( answer_1, @@ -624,12 +592,13 @@ def test_service_can_ask_questions_to_multiple_children(self): def test_child_can_ask_its_own_child_questions(self): """Test that a child can contact its own child while answering a question from a parent.""" - child_of_child = self.make_new_child(BACKEND, run_function_returnee=DifferentMockAnalysis()) def child_run_function(analysis_id, input_values, *args, **kwargs): subscription, _ = child.ask(service_id=child_of_child.id, input_values=input_values) return MockAnalysis(output_values={input_values["question"]: child.wait_for_answer(subscription)}) + child_of_child = self.make_new_child(BACKEND, run_function_returnee=DifferentMockAnalysis()) + child = MockService( backend=BACKEND, run_function=child_run_function, @@ -638,16 +607,15 @@ def child_run_function(analysis_id, input_values, *args, **kwargs): parent = MockService(backend=BACKEND, children={child.id: child}) - with self.service_patcher: - child.serve() - child_of_child.serve() + child.serve() + child_of_child.serve() - subscription, _ = parent.ask( - service_id=child.id, - input_values={"question": "What does the child of the child say?"}, - ) + subscription, _ = parent.ask( + service_id=child.id, + input_values={"question": "What does the child of the child say?"}, + ) - answer = parent.wait_for_answer(subscription) + answer = parent.wait_for_answer(subscription) self.assertEqual( answer, @@ -664,8 +632,6 @@ def child_run_function(analysis_id, input_values, *args, **kwargs): def test_child_can_ask_its_own_children_questions(self): """Test that a child can contact more than one of its own children while answering a question from a parent.""" - first_child_of_child = self.make_new_child(BACKEND, run_function_returnee=DifferentMockAnalysis()) - second_child_of_child = self.make_new_child(BACKEND, run_function_returnee=MockAnalysis()) def child_run_function(analysis_id, input_values, *args, **kwargs): subscription_1, _ = child.ask(service_id=first_child_of_child.id, input_values=input_values) @@ -678,25 +644,30 @@ def child_run_function(analysis_id, input_values, *args, **kwargs): } ) + first_child_of_child = self.make_new_child(BACKEND, run_function_returnee=DifferentMockAnalysis()) + second_child_of_child = self.make_new_child(BACKEND, run_function_returnee=MockAnalysis()) + child = MockService( backend=BACKEND, run_function=child_run_function, - children={first_child_of_child.id: first_child_of_child, second_child_of_child.id: second_child_of_child}, + children={ + first_child_of_child.id: first_child_of_child, + second_child_of_child.id: second_child_of_child, + }, ) parent = MockService(backend=BACKEND, children={child.id: child}) - with self.service_patcher: - child.serve() - first_child_of_child.serve() - second_child_of_child.serve() + child.serve() + first_child_of_child.serve() + second_child_of_child.serve() - subscription, _ = parent.ask( - service_id=child.id, - input_values={"question": "What does the child of the child say?"}, - ) + subscription, _ = parent.ask( + service_id=child.id, + input_values={"question": "What does the child of the child say?"}, + ) - answer = parent.wait_for_answer(subscription) + answer = parent.wait_for_answer(subscription) self.assertEqual( answer, @@ -719,35 +690,32 @@ def test_child_messages_can_be_recorded_by_parent(self): """Test that the parent can record messages it receives from its child to a JSON file.""" child = MockService(backend=BACKEND, run_function=self.create_run_function()) parent = MockService(backend=BACKEND, children={child.id: child}) + child.serve() - with self.service_patcher: - child.serve() - subscription, _ = parent.ask(service_id=child.id, input_values={}, subscribe_to_logs=True) - parent.wait_for_answer(subscription) + subscription, _ = parent.ask(service_id=child.id, input_values={}, subscribe_to_logs=True) + parent.wait_for_answer(subscription) # Check that the child's messages have been recorded by the parent. - self.assertEqual(parent.received_messages[0]["kind"], "delivery_acknowledgement") - self.assertEqual(parent.received_messages[1]["kind"], "log_record") - self.assertEqual(parent.received_messages[2]["kind"], "log_record") - self.assertEqual(parent.received_messages[3]["kind"], "log_record") - self.assertEqual(parent.received_messages[4], {"kind": "result", "output_values": "Hello! It worked!"}) + self.assertEqual(parent.received_events[0]["kind"], "delivery_acknowledgement") + self.assertEqual(parent.received_events[1]["kind"], "log_record") + self.assertEqual(parent.received_events[2]["kind"], "log_record") + self.assertEqual(parent.received_events[3]["kind"], "log_record") + self.assertEqual(parent.received_events[4], {"kind": "result", "output_values": "Hello! It worked!"}) def test_child_exception_message_can_be_recorded_by_parent(self): """Test that the parent can record exceptions raised by the child.""" child = self.make_new_child_with_error(ValueError("Oh no.")) parent = MockService(backend=BACKEND, children={child.id: child}) + child.serve() - with self.service_patcher: - child.serve() - - with self.assertRaises(ValueError): - subscription, _ = parent.ask(service_id=child.id, input_values={}, subscribe_to_logs=True) - parent.wait_for_answer(subscription) + with self.assertRaises(ValueError): + subscription, _ = parent.ask(service_id=child.id, input_values={}, subscribe_to_logs=True) + parent.wait_for_answer(subscription) # Check that the child's messages have been recorded by the parent. - self.assertEqual(parent.received_messages[0]["kind"], "delivery_acknowledgement") - self.assertEqual(parent.received_messages[1]["kind"], "exception") - self.assertIn("Oh no.", parent.received_messages[1]["exception_message"]) + self.assertEqual(parent.received_events[0]["kind"], "delivery_acknowledgement") + self.assertEqual(parent.received_events[1]["kind"], "exception") + self.assertIn("Oh no.", parent.received_events[1]["exception_message"]) def test_child_sends_heartbeat_messages_at_expected_regular_intervals(self): """Test that children send heartbeat messages at the expected regular intervals.""" @@ -759,28 +727,26 @@ def run_function(*args, **kwargs): child = MockService(backend=BACKEND, run_function=lambda *args, **kwargs: run_function()) parent = MockService(backend=BACKEND, children={child.id: child}) + child.serve() - with self.service_patcher: - child.serve() - - with patch( - "octue.cloud.emulators._pub_sub.MockService.answer", - functools.partial(child.answer, heartbeat_interval=expected_interval), - ): - subscription, _ = parent.ask( - service_id=child.id, - input_values={}, - subscribe_to_logs=True, - save_diagnostics="SAVE_DIAGNOSTICS_ON_CRASH", - ) + with patch( + "octue.cloud.emulators._pub_sub.MockService.answer", + functools.partial(child.answer, heartbeat_interval=expected_interval), + ): + subscription, _ = parent.ask( + service_id=child.id, + input_values={}, + subscribe_to_logs=True, + save_diagnostics="SAVE_DIAGNOSTICS_ON_CRASH", + ) - parent.wait_for_answer(subscription) + parent.wait_for_answer(subscription) - self.assertEqual(parent.received_messages[1]["kind"], "heartbeat") - self.assertEqual(parent.received_messages[2]["kind"], "heartbeat") + self.assertEqual(parent.received_events[1]["kind"], "heartbeat") + self.assertEqual(parent.received_events[2]["kind"], "heartbeat") - first_heartbeat_time = datetime.datetime.fromisoformat(parent.received_messages[1]["datetime"]) - second_heartbeat_time = datetime.datetime.fromisoformat(parent.received_messages[2]["datetime"]) + first_heartbeat_time = datetime.datetime.fromisoformat(parent.received_events[1]["datetime"]) + second_heartbeat_time = datetime.datetime.fromisoformat(parent.received_events[2]["datetime"]) self.assertAlmostEqual( second_heartbeat_time - first_heartbeat_time, @@ -806,19 +772,17 @@ def run_function(*args, **kwargs): child = MockService(backend=BACKEND, run_function=run_function) parent = MockService(backend=BACKEND, children={child.id: child}) + child.serve() - with self.service_patcher: - child.serve() - - subscription, _ = parent.ask( - service_id=child.id, - input_values={}, - subscribe_to_logs=True, - save_diagnostics="SAVE_DIAGNOSTICS_ON_CRASH", - ) + subscription, _ = parent.ask( + service_id=child.id, + input_values={}, + subscribe_to_logs=True, + save_diagnostics="SAVE_DIAGNOSTICS_ON_CRASH", + ) - monitor_messages = [] - result = parent.wait_for_answer(subscription, handle_monitor_message=monitor_messages.append) + monitor_messages = [] + result = parent.wait_for_answer(subscription, handle_monitor_message=monitor_messages.append) # Check that multiple monitor messages were sent and received. self.assertTrue(len(monitor_messages) > 1) @@ -886,6 +850,10 @@ def mock_child_app(analysis): children={child.id: child}, ) + static_child_of_child.serve() + dynamic_child_of_child.serve() + child.serve() + dynamic_children = [ { "key": "expected_child", @@ -894,13 +862,8 @@ def mock_child_app(analysis): }, ] - with self.service_patcher: - static_child_of_child.serve() - dynamic_child_of_child.serve() - child.serve() - - subscription, _ = parent.ask(service_id=child.id, input_values={}, children=dynamic_children) - answer = parent.wait_for_answer(subscription) + subscription, _ = parent.ask(service_id=child.id, input_values={}, children=dynamic_children) + answer = parent.wait_for_answer(subscription) self.assertEqual(answer["output_values"], "I am the dynamic child.") diff --git a/tests/cloud/pub_sub/test_subscription.py b/tests/cloud/pub_sub/test_subscription.py index 433f984b7..649ab21f5 100644 --- a/tests/cloud/pub_sub/test_subscription.py +++ b/tests/cloud/pub_sub/test_subscription.py @@ -11,31 +11,19 @@ class TestSubscription(BaseTestCase): - topic = Topic(name="world", project_name="my-project") - subscription = Subscription(name="world", topic=topic, project_name=TEST_PROJECT_NAME) + topic = Topic(name="world", project_name=TEST_PROJECT_NAME) + subscription = Subscription(name="world", topic=topic) def test_repr(self): """Test that subscriptions are represented correctly.""" - self.assertEqual(repr(self.subscription), "") - - def test_namespace_only_in_name_once(self): - """Test that the subscription's namespace only appears in its name once, even if it is repeated.""" - self.assertEqual(self.subscription.name, "octue.services.world") - - subscription_with_repeated_namespace = Subscription( - name="octue.services.world", - topic=self.topic, - project_name=TEST_PROJECT_NAME, - ) - - self.assertEqual(subscription_with_repeated_namespace.name, "octue.services.world") + self.assertEqual(repr(self.subscription), "") def test_create_without_allow_existing_when_subscription_already_exists(self): """Test that an error is raised when trying to create a subscription that already exists and `allow_existing` is `False`. """ with patch("octue.cloud.pub_sub.subscription.SubscriberClient", MockSubscriber): - subscription = Subscription(name="world", topic=self.topic, project_name=TEST_PROJECT_NAME) + subscription = Subscription(name="world", topic=self.topic) with patch( "octue.cloud.emulators._pub_sub.MockSubscriber.create_subscription", @@ -52,7 +40,7 @@ def test_create_with_allow_existing_when_already_exists(self): error. """ with patch("octue.cloud.pub_sub.subscription.SubscriberClient", MockSubscriber): - subscription = Subscription(name="world", topic=self.topic, project_name=TEST_PROJECT_NAME) + subscription = Subscription(name="world", topic=self.topic) with patch( "octue.cloud.emulators._pub_sub.MockSubscriber.create_subscription", @@ -69,13 +57,7 @@ def test_create_pull_subscription(self): """ project_name = os.environ["TEST_PROJECT_NAME"] topic = Topic(name="my-topic", project_name=project_name) - - subscription = Subscription( - name="world", - topic=topic, - project_name=project_name, - filter='attributes.question_uuid = "abc"', - ) + subscription = Subscription(name="world", topic=topic, filter='attributes.question_uuid = "abc"') for allow_existing in (True, False): with self.subTest(allow_existing=allow_existing): @@ -96,13 +78,7 @@ def test_create_push_subscription(self): """Test that creating a push subscription works properly.""" project_name = os.environ["TEST_PROJECT_NAME"] topic = Topic(name="my-topic", project_name=project_name) - - subscription = Subscription( - name="world", - topic=topic, - project_name=project_name, - push_endpoint="https://example.com/endpoint", - ) + subscription = Subscription(name="world", topic=topic, push_endpoint="https://example.com/endpoint") with patch("google.pubsub_v1.SubscriberClient.create_subscription", new=MockSubscriptionCreationResponse): response = subscription.create(allow_existing=True) @@ -115,18 +91,12 @@ def test_create_push_subscription(self): self.assertEqual(response._pb.push_config.push_endpoint, "https://example.com/endpoint") def test_is_pull_subscription(self): - """Test that `is_pull_subscription` is `True` and `is_push_subscription` is `False` for a pull subscription.""" + """Test that `is_pull_subscription` is `True` for a pull subscription.""" self.assertTrue(self.subscription.is_pull_subscription) self.assertFalse(self.subscription.is_push_subscription) def test_is_push_subscription(self): - """Test that `is_pull_subscription` is `False` and `is_push_subscription` is `True` for a pull subscription.""" - push_subscription = Subscription( - name="world", - topic=self.topic, - project_name=TEST_PROJECT_NAME, - push_endpoint="https://example.com/endpoint", - ) - + """Test that `is_push_subscription` is `True` for a push subscription.""" + push_subscription = Subscription(name="world", topic=self.topic, push_endpoint="https://example.com/endpoint") self.assertTrue(push_subscription.is_push_subscription) self.assertFalse(push_subscription.is_pull_subscription) diff --git a/tests/cloud/pub_sub/test_topic.py b/tests/cloud/pub_sub/test_topic.py index 03fb05df6..5267226da 100644 --- a/tests/cloud/pub_sub/test_topic.py +++ b/tests/cloud/pub_sub/test_topic.py @@ -10,15 +10,7 @@ class TestTopic(BaseTestCase): def test_repr(self): """Test that Topics are represented correctly.""" topic = Topic(name="world", project_name="my-project") - self.assertEqual(repr(topic), "") - - def test_namespace_only_in_name_once(self): - """Test that the topic's namespace only appears in its name once, even if it is repeated.""" - topic = Topic(name="world", project_name="my-project") - self.assertEqual(topic.name, "octue.services.world") - - topic_with_repeated_namespace = Topic(name="octue.services.world", project_name="my-project") - self.assertEqual(topic_with_repeated_namespace.name, "octue.services.world") + self.assertEqual(repr(topic), "") def test_create(self): """Test that a topic can be created and that it's marked as having its creation triggered locally.""" diff --git a/tests/cloud/test_service_id.py b/tests/cloud/test_service_id.py index b820e35ac..606e76dfa 100644 --- a/tests/cloud/test_service_id.py +++ b/tests/cloud/test_service_id.py @@ -10,6 +10,7 @@ from octue.cloud.service_id import ( convert_service_id_to_pub_sub_form, get_default_sruid, + get_sruid_from_pub_sub_resource_name, get_sruid_parts, raise_if_revision_not_registered, split_service_id, @@ -83,7 +84,6 @@ def test_convert_service_id_to_pub_sub_form(self): ("octue/my-service", "octue.my-service"), ("octue/my-service:0.1.7", "octue.my-service.0-1-7"), ("my-service:3.1.9", "my-service.3-1-9"), - ("octue.services.octue/my-service:0.1.7", "octue.services.octue.my-service.0-1-7"), ) for service_id, pub_sub_service_id in service_ids: @@ -91,6 +91,13 @@ def test_convert_service_id_to_pub_sub_form(self): self.assertEqual(convert_service_id_to_pub_sub_form(service_id), pub_sub_service_id) +class TestGetSRUIDFromPubSubResourceName(unittest.TestCase): + def test_get_sruid_from_pub_sub_resource_name(self): + """Test that an SRUID can be extracted from a Pub/Sub resource name.""" + sruid = get_sruid_from_pub_sub_resource_name("octue.example-service-cloud-run.0-3-2") + self.assertEqual(sruid, "octue/example-service-cloud-run:0.3.2") + + class TestValidateSRUID(unittest.TestCase): def test_error_raised_if_service_id_invalid(self): """Test that an error is raised if an invalid SRUID is given.""" diff --git a/tests/data/diagnostics/questions.json b/tests/data/diagnostics/questions.json index 212ae9b36..44edac3d7 100644 --- a/tests/data/diagnostics/questions.json +++ b/tests/data/diagnostics/questions.json @@ -1,7 +1,7 @@ [ { "id": "octue/my-child:2.3.0", - "messages": [ + "events": [ { "kind": "log_record", "log_record": { "msg": "Starting analysis." } diff --git a/tests/data/events.json b/tests/data/events.json new file mode 100644 index 000000000..b170a47e7 --- /dev/null +++ b/tests/data/events.json @@ -0,0 +1,259 @@ +[ + { + "event": { + "datetime": "2024-03-06T15:44:18.156044", + "kind": "delivery_acknowledgement" + }, + "attributes": { + "datetime": "2024-04-11T10:46:48.236064", + "uuid": "a9de11b1-e88f-43fa-b3a4-40a590c3443f", + "order": "0", + "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", + "originator": "octue/test-service:1.0.0", + "sender": "octue/test-service:1.0.0", + "sender_type": "CHILD", + "sender_sdk_version": "0.51.0", + "recipient": "octue/another-service:3.2.1" + } + }, + { + "event": { + "kind": "log_record", + "log_record": { + "args": null, + "created": 1709739858.1877987, + "exc_info": null, + "exc_text": null, + "filename": "app.py", + "funcName": "run", + "levelname": "INFO", + "levelno": 20, + "lineno": 15, + "module": "app", + "msecs": 187.79873847961426, + "msg": "Started example analysis.", + "name": "app", + "pathname": "/workspace/example_service_cloud_run/app.py", + "process": 2, + "processName": "MainProcess", + "relativeCreated": 5152.963876724243, + "stack_info": null, + "thread": 68328473233152, + "threadName": "ThreadPoolExecutor-0_2" + } + }, + "attributes": { + "datetime": "2024-04-11T10:46:48.236064", + "uuid": "a9de11b1-e88f-43fa-b3a4-40a590c3443f", + "order": "1", + "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", + "originator": "octue/test-service:1.0.0", + "sender": "octue/test-service:1.0.0", + "sender_type": "CHILD", + "sender_sdk_version": "0.51.0", + "recipient": "octue/another-service:3.2.1" + } + }, + { + "event": { + "kind": "log_record", + "log_record": { + "args": null, + "created": 1709739858.190219, + "exc_info": null, + "exc_text": null, + "filename": "submodule.py", + "funcName": "do_something", + "levelname": "INFO", + "levelno": 20, + "lineno": 8, + "module": "submodule", + "msecs": 190.21892547607422, + "msg": "Submodules are included in logging.", + "name": "example_service_cloud_run.submodule", + "pathname": "/workspace/example_service_cloud_run/submodule.py", + "process": 2, + "processName": "MainProcess", + "relativeCreated": 5155.384063720703, + "stack_info": null, + "thread": 68328473233152, + "threadName": "ThreadPoolExecutor-0_2" + } + }, + "attributes": { + "datetime": "2024-04-11T10:46:48.236064", + "uuid": "a9de11b1-e88f-43fa-b3a4-40a590c3443f", + "order": "2", + "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", + "originator": "octue/test-service:1.0.0", + "sender": "octue/test-service:1.0.0", + "sender_type": "CHILD", + "sender_sdk_version": "0.51.0", + "recipient": "octue/another-service:3.2.1" + } + }, + { + "event": { + "kind": "log_record", + "log_record": { + "args": null, + "created": 1709739860.2234063, + "exc_info": null, + "exc_text": null, + "filename": "analysis.py", + "funcName": "finalise", + "levelname": "INFO", + "levelno": 20, + "lineno": 158, + "module": "analysis", + "msecs": 223.40631484985352, + "msg": "Validated output values and output manifest against the twine.", + "name": "octue.resources.analysis", + "pathname": "/usr/local/lib/python3.9/site-packages/octue/resources/analysis.py", + "process": 2, + "processName": "MainProcess", + "relativeCreated": 7188.571453094482, + "stack_info": null, + "thread": 68328473233152, + "threadName": "ThreadPoolExecutor-0_2" + } + }, + "attributes": { + "datetime": "2024-04-11T10:46:48.236064", + "uuid": "a9de11b1-e88f-43fa-b3a4-40a590c3443f", + "order": "3", + "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", + "originator": "octue/test-service:1.0.0", + "sender": "octue/test-service:1.0.0", + "sender_type": "CHILD", + "sender_sdk_version": "0.51.0", + "recipient": "octue/another-service:3.2.1" + } + }, + { + "event": { + "kind": "log_record", + "log_record": { + "args": null, + "created": 1709739861.5923417, + "exc_info": null, + "exc_text": null, + "filename": "analysis.py", + "funcName": "finalise", + "levelname": "INFO", + "levelno": 20, + "lineno": 174, + "module": "analysis", + "msecs": 592.3416614532471, + "msg": "Uploaded output datasets to 'gs://octue-sdk-python-test-bucket/example_output_datasets'.", + "name": "octue.resources.analysis", + "pathname": "/usr/local/lib/python3.9/site-packages/octue/resources/analysis.py", + "process": 2, + "processName": "MainProcess", + "relativeCreated": 8557.506799697876, + "stack_info": null, + "thread": 68328473233152, + "threadName": "ThreadPoolExecutor-0_2" + } + }, + "attributes": { + "datetime": "2024-04-11T10:46:48.236064", + "uuid": "a9de11b1-e88f-43fa-b3a4-40a590c3443f", + "order": "4", + "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", + "originator": "octue/test-service:1.0.0", + "sender": "octue/test-service:1.0.0", + "sender_type": "CHILD", + "sender_sdk_version": "0.51.0", + "recipient": "octue/another-service:3.2.1" + } + }, + { + "event": { + "kind": "log_record", + "log_record": { + "args": null, + "created": 1709739861.5949728, + "exc_info": null, + "exc_text": null, + "filename": "app.py", + "funcName": "run", + "levelname": "INFO", + "levelno": 20, + "lineno": 28, + "module": "app", + "msecs": 594.9728488922119, + "msg": "Finished example analysis.", + "name": "app", + "pathname": "/workspace/example_service_cloud_run/app.py", + "process": 2, + "processName": "MainProcess", + "relativeCreated": 8560.13798713684, + "stack_info": null, + "thread": 68328473233152, + "threadName": "ThreadPoolExecutor-0_2" + } + }, + "attributes": { + "datetime": "2024-04-11T10:46:48.236064", + "uuid": "a9de11b1-e88f-43fa-b3a4-40a590c3443f", + "order": "5", + "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", + "originator": "octue/test-service:1.0.0", + "sender": "octue/test-service:1.0.0", + "sender_type": "CHILD", + "sender_sdk_version": "0.51.0", + "recipient": "octue/another-service:3.2.1" + } + }, + { + "event": { + "kind": "result", + "output_manifest": { + "datasets": { + "example_dataset": { + "files": [ + "gs://octue-sdk-python-test-bucket/example_output_datasets/example_dataset/output.dat" + ], + "id": "419bff6b-08c3-4c16-9eb1-5d1709168003", + "labels": [], + "name": "divergent-strange-gharial-of-pizza", + "path": "https://storage.googleapis.com/octue-sdk-python-test-bucket/example_output_datasets/example_dataset/.signed_metadata_files/divergent-strange-gharial-of-pizza", + "tags": {} + } + }, + "id": "a13713ae-f207-41c6-9e29-0a848ced6039", + "name": null + }, + "output_values": [1, 2, 3, 4, 5] + }, + "attributes": { + "datetime": "2024-04-11T10:46:48.236064", + "uuid": "a9de11b1-e88f-43fa-b3a4-40a590c3443f", + "order": "6", + "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", + "originator": "octue/test-service:1.0.0", + "sender": "octue/test-service:1.0.0", + "sender_type": "CHILD", + "sender_sdk_version": "0.51.0", + "recipient": "octue/another-service:3.2.1" + } + }, + { + "event": { + "datetime": "2024-03-06T15:46:18.167424", + "kind": "heartbeat" + }, + "attributes": { + "datetime": "2024-04-11T10:46:48.236064", + "uuid": "a9de11b1-e88f-43fa-b3a4-40a590c3443f", + "order": "7", + "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", + "originator": "octue/test-service:1.0.0", + "sender": "octue/test-service:1.0.0", + "sender_type": "CHILD", + "sender_sdk_version": "0.51.0", + "recipient": "octue/another-service:3.2.1" + } + } +] diff --git a/tests/resources/test_child.py b/tests/resources/test_child.py index 1e9c98234..c0e866736 100644 --- a/tests/resources/test_child.py +++ b/tests/resources/test_child.py @@ -8,7 +8,7 @@ from google.auth.exceptions import DefaultCredentialsError -from octue.cloud.emulators._pub_sub import MockAnalysis, MockService, MockSubscriber, MockSubscription, MockTopic +from octue.cloud.emulators._pub_sub import MockAnalysis, MockService from octue.cloud.emulators.child import ServicePatcher from octue.resources.child import Child from octue.resources.service_backends import GCPPubSubBackend @@ -32,6 +32,24 @@ def mock_run_function_that_fails_every_other_time(analysis_id, input_values, *ar class TestChild(BaseTestCase): + service_patcher = ServicePatcher() + + @classmethod + def setUpClass(cls): + """Start the service patcher.. + + :return None: + """ + cls.service_patcher.start() + + @classmethod + def tearDownClass(cls): + """Stop the services patcher. + + :return None: + """ + cls.service_patcher.stop() + def test_representation(self): """Test that children are represented correctly as a string.""" self.assertEqual( @@ -55,18 +73,14 @@ def test_instantiating_child_without_credentials(self): def test_child_cannot_be_asked_question_without_credentials(self): """Test that a child cannot be asked a question without Google Cloud credentials being available.""" with patch.dict(os.environ, clear=True): - with patch("octue.cloud.pub_sub.service.Topic", new=MockTopic): - with patch("octue.cloud.pub_sub.service.Subscription", new=MockSubscription): - with patch("octue.resources.child.BACKEND_TO_SERVICE_MAPPING", {"GCPPubSubBackend": MockService}): - with patch("google.cloud.pubsub_v1.SubscriberClient", new=MockSubscriber): - - child = Child( - id=f"octue/my-child:{MOCK_SERVICE_REVISION_TAG}", - backend={"name": "GCPPubSubBackend", "project_name": "blah"}, - ) + with patch("octue.resources.child.BACKEND_TO_SERVICE_MAPPING", {"GCPPubSubBackend": MockService}): + child = Child( + id=f"octue/my-child:{MOCK_SERVICE_REVISION_TAG}", + backend={"name": "GCPPubSubBackend", "project_name": "blah"}, + ) - with self.assertRaises(DefaultCredentialsError): - child.ask({"some": "input"}) + with self.assertRaises(DefaultCredentialsError): + child.ask({"some": "input"}) def test_child_can_be_asked_multiple_questions(self): """Test that a child can be asked multiple questions.""" @@ -76,16 +90,14 @@ def mock_run_function(analysis_id, input_values, *args, **kwargs): responding_service = MockService(backend=GCPPubSubBackend(project_name="blah"), run_function=mock_run_function) - with ServicePatcher(): - with patch("octue.resources.child.BACKEND_TO_SERVICE_MAPPING", {"GCPPubSubBackend": MockService}): - responding_service.serve() - - child = Child(id=responding_service.id, backend={"name": "GCPPubSubBackend", "project_name": "blah"}) + with patch("octue.resources.child.BACKEND_TO_SERVICE_MAPPING", {"GCPPubSubBackend": MockService}): + responding_service.serve() + child = Child(id=responding_service.id, backend={"name": "GCPPubSubBackend", "project_name": "blah"}) - # Make sure the child's underlying mock service knows how to access the mock responding service. - child._service.children[responding_service.id] = responding_service - self.assertEqual(child.ask([1, 2, 3, 4])["output_values"], [1, 2, 3, 4]) - self.assertEqual(child.ask([5, 6, 7, 8])["output_values"], [5, 6, 7, 8]) + # Make sure the child's underlying mock service knows how to access the mock responding service. + child._service.children[responding_service.id] = responding_service + self.assertEqual(child.ask([1, 2, 3, 4])["output_values"], [1, 2, 3, 4]) + self.assertEqual(child.ask([5, 6, 7, 8])["output_values"], [5, 6, 7, 8]) def test_ask_multiple(self): """Test that a child can be asked multiple questions in parallel and return the answers in the correct order.""" @@ -96,27 +108,26 @@ def mock_run_function(analysis_id, input_values, *args, **kwargs): responding_service = MockService(backend=GCPPubSubBackend(project_name="blah"), run_function=mock_run_function) - with ServicePatcher(): - with patch("octue.resources.child.BACKEND_TO_SERVICE_MAPPING", {"GCPPubSubBackend": MockService}): - responding_service.serve() + with patch("octue.resources.child.BACKEND_TO_SERVICE_MAPPING", {"GCPPubSubBackend": MockService}): + responding_service.serve() - child = Child(id=responding_service.id, backend={"name": "GCPPubSubBackend", "project_name": "blah"}) + child = Child(id=responding_service.id, backend={"name": "GCPPubSubBackend", "project_name": "blah"}) - # Make sure the child's underlying mock service knows how to access the mock responding service. - child._service.children[responding_service.id] = responding_service + # Make sure the child's underlying mock service knows how to access the mock responding service. + child._service.children[responding_service.id] = responding_service - answers = child.ask_multiple( - {"input_values": [1, 2, 3, 4]}, - {"input_values": [5, 6, 7, 8]}, - ) + answers = child.ask_multiple( + {"input_values": [1, 2, 3, 4]}, + {"input_values": [5, 6, 7, 8]}, + ) - self.assertEqual( - answers, - [ - {"output_values": [1, 2, 3, 4], "output_manifest": None}, - {"output_values": [5, 6, 7, 8], "output_manifest": None}, - ], - ) + self.assertEqual( + answers, + [ + {"output_values": [1, 2, 3, 4], "output_manifest": None}, + {"output_values": [5, 6, 7, 8], "output_manifest": None}, + ], + ) def test_error_raised_in_ask_multiple_if_one_question_fails_when_raise_errors_is_true(self): """Test that an error is raised if any of the questions given to `Child.ask_multiple` fail when `raise_errors` @@ -137,21 +148,20 @@ def mock_run_function_that_sometimes_fails(analysis_id, input_values, *args, **k run_function=functools.partial(mock_run_function_that_sometimes_fails, runs=Value("d", 0)), ) - with ServicePatcher(): - with patch("octue.resources.child.BACKEND_TO_SERVICE_MAPPING", {"GCPPubSubBackend": MockService}): - responding_service.serve() + with patch("octue.resources.child.BACKEND_TO_SERVICE_MAPPING", {"GCPPubSubBackend": MockService}): + responding_service.serve() - child = Child(id=responding_service.id, backend={"name": "GCPPubSubBackend", "project_name": "blah"}) + child = Child(id=responding_service.id, backend={"name": "GCPPubSubBackend", "project_name": "blah"}) - # Make sure the child's underlying mock service knows how to access the mock responding service. - child._service.children[responding_service.id] = responding_service + # Make sure the child's underlying mock service knows how to access the mock responding service. + child._service.children[responding_service.id] = responding_service - with self.assertRaises(ValueError): - child.ask_multiple( - {"input_values": [1, 2, 3, 4]}, - {"input_values": [5, 6, 7, 8]}, - {"input_values": [9, 10, 11, 12]}, - ) + with self.assertRaises(ValueError): + child.ask_multiple( + {"input_values": [1, 2, 3, 4]}, + {"input_values": [5, 6, 7, 8]}, + {"input_values": [9, 10, 11, 12]}, + ) def test_error_not_raised_by_ask_multiple_if_one_question_fails_when_raise_errors_is_false(self): """Test that an error is not raised if one of the questions given to `Child.ask_multiple` fail when @@ -162,21 +172,20 @@ def test_error_not_raised_by_ask_multiple_if_one_question_fails_when_raise_error run_function=functools.partial(mock_run_function_that_fails_every_other_time, runs=Value("d", 0)), ) - with ServicePatcher(): - with patch("octue.resources.child.BACKEND_TO_SERVICE_MAPPING", {"GCPPubSubBackend": MockService}): - responding_service.serve() + with patch("octue.resources.child.BACKEND_TO_SERVICE_MAPPING", {"GCPPubSubBackend": MockService}): + responding_service.serve() - child = Child(id=responding_service.id, backend={"name": "GCPPubSubBackend", "project_name": "blah"}) + child = Child(id=responding_service.id, backend={"name": "GCPPubSubBackend", "project_name": "blah"}) - # Make sure the child's underlying mock service knows how to access the mock responding service. - child._service.children[responding_service.id] = responding_service + # Make sure the child's underlying mock service knows how to access the mock responding service. + child._service.children[responding_service.id] = responding_service - answers = child.ask_multiple( - {"input_values": [1, 2, 3, 4]}, - {"input_values": [5, 6, 7, 8]}, - {"input_values": [9, 10, 11, 12]}, - raise_errors=False, - ) + answers = child.ask_multiple( + {"input_values": [1, 2, 3, 4]}, + {"input_values": [5, 6, 7, 8]}, + {"input_values": [9, 10, 11, 12]}, + raise_errors=False, + ) successful_answers = [] failed_answers = [] @@ -204,22 +213,21 @@ def test_ask_multiple_with_failed_question_retry(self): run_function=functools.partial(mock_run_function_that_fails_every_other_time, runs=Value("d", 0)), ) - with ServicePatcher(): - with patch("octue.resources.child.BACKEND_TO_SERVICE_MAPPING", {"GCPPubSubBackend": MockService}): - responding_service.serve() + with patch("octue.resources.child.BACKEND_TO_SERVICE_MAPPING", {"GCPPubSubBackend": MockService}): + responding_service.serve() - child = Child(id=responding_service.id, backend={"name": "GCPPubSubBackend", "project_name": "blah"}) + child = Child(id=responding_service.id, backend={"name": "GCPPubSubBackend", "project_name": "blah"}) - # Make sure the child's underlying mock service knows how to access the mock responding service. - child._service.children[responding_service.id] = responding_service + # Make sure the child's underlying mock service knows how to access the mock responding service. + child._service.children[responding_service.id] = responding_service - # Only ask two questions so the question success/failure order plays out as desired. - answers = child.ask_multiple( - {"input_values": [1, 2, 3, 4]}, - {"input_values": [5, 6, 7, 8]}, - raise_errors=False, - max_retries=1, - ) + # Only ask two questions so the question success/failure order plays out as desired. + answers = child.ask_multiple( + {"input_values": [1, 2, 3, 4]}, + {"input_values": [5, 6, 7, 8]}, + raise_errors=False, + max_retries=1, + ) # Check that both questions succeeded. self.assertEqual( @@ -247,24 +255,23 @@ def test_ask_multiple_with_multiple_failed_question_retries(self): run_function=functools.partial(mock_run_function_that_fails_every_other_time, runs=Value("d", 0)), ) - with ServicePatcher(): - with patch("octue.resources.child.BACKEND_TO_SERVICE_MAPPING", {"GCPPubSubBackend": MockService}): - responding_service.serve() + with patch("octue.resources.child.BACKEND_TO_SERVICE_MAPPING", {"GCPPubSubBackend": MockService}): + responding_service.serve() - child = Child(id=responding_service.id, backend={"name": "GCPPubSubBackend", "project_name": "blah"}) + child = Child(id=responding_service.id, backend={"name": "GCPPubSubBackend", "project_name": "blah"}) - # Make sure the child's underlying mock service knows how to access the mock responding service. - child._service.children[responding_service.id] = responding_service + # Make sure the child's underlying mock service knows how to access the mock responding service. + child._service.children[responding_service.id] = responding_service - # Only ask two questions so the question success/failure order plays out as desired. - answers = child.ask_multiple( - {"input_values": [1, 2, 3, 4]}, - {"input_values": [5, 6, 7, 8]}, - {"input_values": [9, 10, 11, 12]}, - {"input_values": [13, 14, 15, 16]}, - raise_errors=False, - max_retries=2, - ) + # Only ask two questions so the question success/failure order plays out as desired. + answers = child.ask_multiple( + {"input_values": [1, 2, 3, 4]}, + {"input_values": [5, 6, 7, 8]}, + {"input_values": [9, 10, 11, 12]}, + {"input_values": [13, 14, 15, 16]}, + raise_errors=False, + max_retries=2, + ) # Check that all four questions succeeded. self.assertEqual( @@ -284,23 +291,22 @@ def test_ask_multiple_with_prevented_retries(self): run_function=functools.partial(mock_run_function_that_fails_every_other_time, runs=Value("d", 0)), ) - with ServicePatcher(): - with patch("octue.resources.child.BACKEND_TO_SERVICE_MAPPING", {"GCPPubSubBackend": MockService}): - responding_service.serve() + with patch("octue.resources.child.BACKEND_TO_SERVICE_MAPPING", {"GCPPubSubBackend": MockService}): + responding_service.serve() - child = Child(id=responding_service.id, backend={"name": "GCPPubSubBackend", "project_name": "blah"}) + child = Child(id=responding_service.id, backend={"name": "GCPPubSubBackend", "project_name": "blah"}) - # Make sure the child's underlying mock service knows how to access the mock responding service. - child._service.children[responding_service.id] = responding_service + # Make sure the child's underlying mock service knows how to access the mock responding service. + child._service.children[responding_service.id] = responding_service - # Only ask two questions so the question success/failure order plays out as desired. - answers = child.ask_multiple( - {"input_values": [1, 2, 3, 4]}, - {"input_values": [5, 6, 7, 8]}, - raise_errors=False, - max_retries=1, - prevent_retries_when=[ValueError], - ) + # Only ask two questions so the question success/failure order plays out as desired. + answers = child.ask_multiple( + {"input_values": [1, 2, 3, 4]}, + {"input_values": [5, 6, 7, 8]}, + raise_errors=False, + max_retries=1, + prevent_retries_when=[ValueError], + ) successful_answers = [] failed_answers = [] diff --git a/tests/resources/test_manifest.py b/tests/resources/test_manifest.py index 49717cdda..00ce48fd9 100644 --- a/tests/resources/test_manifest.py +++ b/tests/resources/test_manifest.py @@ -308,3 +308,51 @@ def test_use_signed_urls_for_datasets_is_idempotent(self): self.assertTrue(manifest.datasets["my_dataset"].path.startswith("http")) self.assertIn(".signed_metadata_files", manifest.datasets["my_dataset"].path) + + def test_download_without_paths(self): + """Test downloading a manifest without specifying the `paths` argument.""" + manifest = Manifest(datasets={"my_dataset": self.create_nested_cloud_dataset()}) + paths = manifest.download() + self.assertEqual(set(os.listdir(paths["my_dataset"])), {"file_1.txt", "file_0.txt", "sub-directory"}) + + def test_download_with_paths(self): + """Test downloading a manifest while specifying the `paths` argument.""" + manifest = Manifest( + datasets={ + "dataset1": self.create_nested_cloud_dataset(), + "dataset2": self.create_nested_cloud_dataset(), + }, + ) + + with tempfile.TemporaryDirectory() as temporary_directory: + paths = manifest.download(paths={"dataset1": temporary_directory}) + + # Check that dataset1 has been downloaded to the given directory. + self.assertEqual(paths["dataset1"], temporary_directory) + self.assertEqual(set(os.listdir(paths["dataset1"])), {"file_1.txt", "file_0.txt", "sub-directory"}) + + # Check that dataset2 has been downloaded to a temporary directory. + self.assertNotEqual(paths["dataset2"], temporary_directory) + self.assertEqual(set(os.listdir(paths["dataset2"])), {"file_1.txt", "file_0.txt", "sub-directory"}) + + def test_download_some_datasets_with_paths(self): + """Test downloading a manifest while specifying the `paths` argument and setting `download_all=False`.""" + manifest = Manifest( + datasets={ + "dataset1": self.create_nested_cloud_dataset(), + "dataset2": self.create_nested_cloud_dataset(), + }, + ) + + with tempfile.TemporaryDirectory() as temporary_directory: + with self.assertLogs() as logging_context: + paths = manifest.download(paths={"dataset1": temporary_directory}, download_all=False) + + # Check that dataset1 has been downloaded to the given directory and dataset2 hasn't been downloaded. + self.assertEqual(paths, {"dataset1": temporary_directory}) + self.assertEqual(set(os.listdir(paths["dataset1"])), {"file_1.txt", "file_0.txt", "sub-directory"}) + + self.assertEqual( + logging_context.records[1].message, + "'dataset2' dataset download skipped as its download path wasn't specified.", + ) diff --git a/tests/resources/test_service_backends.py b/tests/resources/test_service_backends.py index e0cabb357..f385ae3c1 100644 --- a/tests/resources/test_service_backends.py +++ b/tests/resources/test_service_backends.py @@ -19,6 +19,6 @@ def test_repr(self): self.assertEqual(repr(GCPPubSubBackend(project_name="hello")), "") def test_error_raised_if_project_name_is_none(self): - """Test that an error is raised if the project name is not given during `GCPPubSubBackend` instantiation.""" + """Test that an error is raised if the project name isn't given during `GCPPubSubBackend` instantiation.""" with self.assertRaises(CloudLocationNotSpecified): GCPPubSubBackend(project_name=None) diff --git a/tests/templates/test_template_apps.py b/tests/templates/test_template_apps.py index df5809710..d5b784268 100644 --- a/tests/templates/test_template_apps.py +++ b/tests/templates/test_template_apps.py @@ -147,7 +147,7 @@ def test_child_services_template_using_emulated_children(self): ChildEmulator( id=f"template-child-services/wind-speed-service:{MOCK_SERVICE_REVISION_TAG}", internal_service_name=runner.service_id, - messages=[ + events=[ {"kind": "log_record", "log_record": {"msg": "This is an emulated child log message."}}, {"kind": "result", "output_values": [10], "output_manifest": None}, ], @@ -155,7 +155,7 @@ def test_child_services_template_using_emulated_children(self): ChildEmulator( id=f"template-child-services/elevation-service:{MOCK_SERVICE_REVISION_TAG}", internal_service_name=runner.service_id, - messages=[ + events=[ {"kind": "result", "output_values": [300], "output_manifest": None}, ], ), diff --git a/tests/test_cli.py b/tests/test_cli.py index a4e553b81..153e2eff0 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -309,7 +309,7 @@ def test_get_diagnostics(self): self.assertEqual(questions[0]["id"], f"octue/my-child:{MOCK_SERVICE_REVISION_TAG}") self.assertEqual( - questions[0]["messages"], + questions[0]["events"], [ {"kind": "log_record", "log_record": {"msg": "Starting analysis."}}, {"kind": "log_record", "log_record": {"msg": "Finishing analysis."}}, @@ -376,7 +376,7 @@ def test_get_diagnostics_with_datasets(self): self.assertEqual(questions[0]["id"], f"octue/my-child:{MOCK_SERVICE_REVISION_TAG}") self.assertEqual( - questions[0]["messages"], + questions[0]["events"], [ {"kind": "log_record", "log_record": {"msg": "Starting analysis."}}, {"kind": "log_record", "log_record": {"msg": "Finishing analysis."}}, @@ -402,8 +402,8 @@ def test_create_push_subscription(self): (["--expiration-time=100"], 100), ): with self.subTest(expiration_time_option=expiration_time_option): - with patch("octue.cli.Topic", new=MockTopic): - with patch("octue.cli.Subscription") as mock_subscription: + with patch("octue.cloud.pub_sub.Topic", new=MockTopic): + with patch("octue.cloud.pub_sub.Subscription") as subscription: result = CliRunner().invoke( octue_cli, [ @@ -420,10 +420,6 @@ def test_create_push_subscription(self): self.assertIsNone(result.exception) self.assertEqual(result.exit_code, 0) - - self.assertEqual(mock_subscription.call_args.kwargs["name"], "octue.example-service.3-5-0") - self.assertEqual( - mock_subscription.call_args.kwargs["push_endpoint"], - "https://example.com/endpoint", - ) - self.assertEqual(mock_subscription.call_args.kwargs["expiration_time"], expected_expiration_time) + self.assertEqual(subscription.call_args.kwargs["name"], "octue.example-service.3-5-0") + self.assertEqual(subscription.call_args.kwargs["push_endpoint"], "https://example.com/endpoint") + self.assertEqual(subscription.call_args.kwargs["expiration_time"], expected_expiration_time) diff --git a/tests/test_runner.py b/tests/test_runner.py index ddb7e040d..86fba1372 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -341,13 +341,13 @@ def app(analysis): emulated_children = [ ChildEmulator( id=f"octue/the-child:{MOCK_SERVICE_REVISION_TAG}", - messages=[ + events=[ {"kind": "result", "output_values": [1, 4, 9, 16], "output_manifest": None}, ], ), ChildEmulator( id=f"octue/yet-another-child:{MOCK_SERVICE_REVISION_TAG}", - messages=[ + events=[ {"kind": "log_record", "log_record": {"msg": "Starting analysis."}}, {"kind": "log_record", "log_record": {"msg": "Finishing analysis."}}, { @@ -383,17 +383,17 @@ def app(analysis): self.assertEqual(questions[0]["key"], "my-child") self.assertEqual(questions[0]["id"], f"octue/the-child:{MOCK_SERVICE_REVISION_TAG}") self.assertEqual(questions[0]["input_values"], [1, 2, 3, 4]) - self.assertEqual(len(questions[0]["messages"]), 2) + self.assertEqual(len(questions[0]["events"]), 2) # Second question. self.assertEqual(questions[1]["key"], "another-child") self.assertEqual(questions[1]["id"], f"octue/yet-another-child:{MOCK_SERVICE_REVISION_TAG}") self.assertEqual(questions[1]["input_values"], "miaow") - self.assertEqual(questions[1]["messages"][1]["kind"], "exception") - self.assertEqual(questions[1]["messages"][1]["exception_type"], "ValueError") + self.assertEqual(questions[1]["events"][1]["kind"], "exception") + self.assertEqual(questions[1]["events"][1]["exception_type"], "ValueError") self.assertEqual( - questions[1]["messages"][1]["exception_message"], + questions[1]["events"][1]["exception_message"], f"Error in : Deliberately raised for " f"testing.", ) @@ -789,13 +789,13 @@ def app(analysis): emulated_children = [ ChildEmulator( id=f"octue/a-child:{MOCK_SERVICE_REVISION_TAG}", - messages=[ + events=[ {"kind": "result", "output_values": [1, 4, 9, 16], "output_manifest": None}, ], ), ChildEmulator( id=f"octue/another-child:{MOCK_SERVICE_REVISION_TAG}", - messages=[ + events=[ {"kind": "log_record", "log_record": {"msg": "Starting analysis."}}, {"kind": "log_record", "log_record": {"msg": "Finishing analysis."}}, {"kind": "result", "output_values": "woof", "output_manifest": None}, @@ -902,14 +902,14 @@ def app(analysis): self.assertEqual(questions[0]["key"], "my-child") self.assertEqual(questions[0]["id"], f"octue/a-child:{MOCK_SERVICE_REVISION_TAG}") self.assertEqual(questions[0]["input_values"], [1, 2, 3, 4]) - self.assertEqual(questions[0]["messages"][1]["output_values"], [1, 4, 9, 16]) - self.assertEqual(len(questions[0]["messages"]), 2) + self.assertEqual(questions[0]["events"][1]["output_values"], [1, 4, 9, 16]) + self.assertEqual(len(questions[0]["events"]), 2) # Second question. self.assertEqual(questions[1]["key"], "another-child") self.assertEqual(questions[1]["id"], f"octue/another-child:{MOCK_SERVICE_REVISION_TAG}") self.assertEqual(questions[1]["input_values"], "miaow") - self.assertEqual(questions[1]["messages"][1]["output_values"], "woof") + self.assertEqual(questions[1]["events"][1]["output_values"], "woof") # This should be 4 but log messages aren't currently being handled by the child emulator correctly. - self.assertEqual(len(questions[1]["messages"]), 2) + self.assertEqual(len(questions[1]["events"]), 2) diff --git a/tests/utils/test_testing.py b/tests/utils/test_testing.py index 26030c20f..eb058132c 100644 --- a/tests/utils/test_testing.py +++ b/tests/utils/test_testing.py @@ -28,7 +28,7 @@ def test_load_test_fixture_from_downloaded_diagnostics(self): self.assertEqual(len(child_emulators), 1) self.assertEqual(child_emulators[0].id, f"octue/my-child:{MOCK_SERVICE_REVISION_TAG}") self.assertEqual( - child_emulators[0].messages[2:], + child_emulators[0].events[2:], [ {"kind": "monitor_message", "data": {"sample": "data"}}, {"kind": "result", "output_values": [1, 2, 3, 4, 5]},