|
| 1 | +""" |
| 2 | +In your worker initialization, you can add the canary workflow. |
| 3 | +""" |
| 4 | + |
| 5 | +from datetime import timedelta |
| 6 | +import asyncio |
| 7 | +import time |
| 8 | + |
| 9 | +from temporalio import activity, workflow |
| 10 | +from temporalio.client import Client |
| 11 | +from temporalio.worker import Worker |
| 12 | + |
| 13 | +from canary.your_workflows import YourWorkflow, your_activity |
| 14 | + |
| 15 | +_SECONDS_BETWEEN_CANARY_CHECKS = 3 |
| 16 | + |
| 17 | + |
| 18 | +@activity.defn |
| 19 | +async def canary_activity() -> None: |
| 20 | + """ |
| 21 | + Here's the activity that can probe your worker and see if it's |
| 22 | + still responsive. |
| 23 | + """ |
| 24 | + t_prev = time.time() |
| 25 | + while True: |
| 26 | + await asyncio.sleep(_SECONDS_BETWEEN_CANARY_CHECKS) |
| 27 | + t_new = time.time() |
| 28 | + delay = t_new - (t_prev + _SECONDS_BETWEEN_CANARY_CHECKS) |
| 29 | + t_prev = t_new |
| 30 | + |
| 31 | + # Log the extra time taken by the event loop to get back after the await |
| 32 | + # If you want, you can turn this into a histogram and show the distribution. |
| 33 | + # maybe you could even put it in your metrics. |
| 34 | + activity.logger.info( |
| 35 | + f"The canary detected {round(delay,4)} seconds of event loop delay." |
| 36 | + ) |
| 37 | + print(f"The canary detected {round(delay,4)} seconds of event loop delay.") |
| 38 | + |
| 39 | + |
| 40 | +@workflow.defn |
| 41 | +class CanaryWorkflow: |
| 42 | + """ |
| 43 | + Here's the workflow that can probe your worker and see if it's |
| 44 | + still responsive. |
| 45 | + """ |
| 46 | + |
| 47 | + @workflow.run |
| 48 | + async def run(self) -> str: |
| 49 | + |
| 50 | + return await workflow.execute_activity( |
| 51 | + canary_activity, |
| 52 | + # these timeouts are going to be tricky because if the event loop |
| 53 | + # is indeed blocked, the heartbeats etc may not behave as expected. |
| 54 | + start_to_close_timeout=timedelta(seconds=60 * 100), |
| 55 | + ) |
| 56 | + |
| 57 | + |
| 58 | +async def main(): |
| 59 | + client = await Client.connect("localhost:7233") |
| 60 | + |
| 61 | + async with Worker( |
| 62 | + client, |
| 63 | + task_queue="canary-task-queue", |
| 64 | + workflows=[CanaryWorkflow, YourWorkflow], |
| 65 | + activities=[canary_activity, your_activity], |
| 66 | + ): |
| 67 | + |
| 68 | + # add this to your code |
| 69 | + await client.execute_workflow( |
| 70 | + CanaryWorkflow.run, |
| 71 | + id="canary", |
| 72 | + task_queue="canary-task-queue", |
| 73 | + ) |
| 74 | + |
| 75 | + |
| 76 | +if __name__ == "__main__": |
| 77 | + asyncio.run(main()) |
0 commit comments