|
| 1 | +import * as wf from '@temporalio/workflow'; |
| 2 | +import type * as activities from './activities'; |
| 3 | +import * as _3rdPartyAsyncMutexLibrary from 'async-mutex'; |
| 4 | +import { |
| 5 | + AssignNodesToJobUpdateInput, |
| 6 | + ClusterManagerState, |
| 7 | + ClusterManagerStateSummary, |
| 8 | + DeleteJobUpdateInput, |
| 9 | +} from './types'; |
| 10 | + |
| 11 | +const { assignNodesToJob, unassignNodesForJob } = wf.proxyActivities<typeof activities>({ |
| 12 | + startToCloseTimeout: '1 minute', |
| 13 | +}); |
| 14 | + |
| 15 | +const { findBadNodes } = wf.proxyActivities<typeof activities>({ |
| 16 | + startToCloseTimeout: '1 minute', |
| 17 | + retry: { |
| 18 | + // This activity is called with the nodexMutex held. We do not retry, since retries would block |
| 19 | + // cluster operations. |
| 20 | + maximumAttempts: 1, |
| 21 | + }, |
| 22 | +}); |
| 23 | + |
| 24 | +// ClusterManagerWorkflow keeps track of the job assignments of a cluster of nodes. It exposes an |
| 25 | +// API to started and shutdown the cluster, to assign jobs to nodes, and to delete jobs. The |
| 26 | +// workflow maps this API to signals and updates. Operations altering node assignments must not |
| 27 | +// interleave (must be serialized), and a standard (non-Temporal-specific) async mutex from a 3rd |
| 28 | +// party library is used to ensure this. |
| 29 | +export class ClusterManager { |
| 30 | + state: ClusterManagerState; |
| 31 | + jobsWithNodesAssigned: Set<string>; |
| 32 | + nodesMutex: _3rdPartyAsyncMutexLibrary.Mutex; |
| 33 | + |
| 34 | + constructor(state?: ClusterManagerState) { |
| 35 | + this.state = state ?? { |
| 36 | + clusterStarted: false, |
| 37 | + clusterShutdown: false, |
| 38 | + nodes: new Map<string, string | null>(), |
| 39 | + maxAssignedNodes: 0, |
| 40 | + }; |
| 41 | + this.jobsWithNodesAssigned = new Set<string>(); |
| 42 | + this.nodesMutex = new _3rdPartyAsyncMutexLibrary.Mutex(); |
| 43 | + } |
| 44 | + |
| 45 | + startCluster(): void { |
| 46 | + this.state.clusterStarted = true; |
| 47 | + for (let i = 0; i < 25; i++) { |
| 48 | + this.state.nodes.set(i.toString(), null); |
| 49 | + } |
| 50 | + wf.log.info('Cluster started'); |
| 51 | + } |
| 52 | + |
| 53 | + async shutDownCluster(): Promise<void> { |
| 54 | + await wf.condition(() => this.state.clusterStarted); |
| 55 | + this.state.clusterShutdown = true; |
| 56 | + wf.log.info('Cluster shutdown'); |
| 57 | + } |
| 58 | + |
| 59 | + async assignNodesToJob(input: AssignNodesToJobUpdateInput): Promise<ClusterManagerStateSummary> { |
| 60 | + await wf.condition(() => this.state.clusterStarted); |
| 61 | + if (this.state.clusterShutdown) { |
| 62 | + // If you want the client to receive a failure, either add an update validator and throw the |
| 63 | + // exception from there, or raise an ApplicationError. Other exceptions in the handler will |
| 64 | + // cause the workflow to keep retrying and get it stuck. |
| 65 | + throw new wf.ApplicationFailure('Cannot assign nodes to a job: Cluster is already shut down'); |
| 66 | + } |
| 67 | + return await this.nodesMutex.runExclusive(async (): Promise<ClusterManagerStateSummary> => { |
| 68 | + // Idempotency guard: do nothing if the job already has nodes assigned. |
| 69 | + if (!new Set(this.state.nodes.values()).has(input.jobName)) { |
| 70 | + const unassignedNodes = this.getUnassignedNodes(); |
| 71 | + if (input.numNodes > unassignedNodes.size) { |
| 72 | + throw new wf.ApplicationFailure( |
| 73 | + `Cannot assign ${input.numNodes} nodes; have only ${unassignedNodes.size} available` |
| 74 | + ); |
| 75 | + } |
| 76 | + const nodesToAssign = Array.from(unassignedNodes).slice(0, input.numNodes); |
| 77 | + // This await would be dangerous without the lock held because it would allow interleaving |
| 78 | + // with the deleteJob and performHealthCheck operations, both of which mutate |
| 79 | + // self.state.nodes. |
| 80 | + await assignNodesToJob({ nodes: nodesToAssign, jobName: input.jobName }); |
| 81 | + for (const node of nodesToAssign) { |
| 82 | + this.state.nodes.set(node, input.jobName); |
| 83 | + } |
| 84 | + this.state.maxAssignedNodes = Math.max(this.state.maxAssignedNodes, this.getAssignedNodes().size); |
| 85 | + } |
| 86 | + return this.getStateSummary(); |
| 87 | + }); |
| 88 | + } |
| 89 | + |
| 90 | + async deleteJob(input: DeleteJobUpdateInput) { |
| 91 | + await wf.condition(() => this.state.clusterStarted); |
| 92 | + if (this.state.clusterShutdown) { |
| 93 | + // If you want the client to receive a failure, either add an update validator and throw the |
| 94 | + // exception from there, or raise an ApplicationError. Other exceptions in the handler will |
| 95 | + // cause the workflow to keep retrying and get it stuck. |
| 96 | + throw new wf.ApplicationFailure('Cannot delete job: Cluster is already shut down'); |
| 97 | + } |
| 98 | + await this.nodesMutex.runExclusive(async () => { |
| 99 | + const nodesToUnassign = Array.from(this.state.nodes.entries()) |
| 100 | + .filter(([_, v]) => v === input.jobName) |
| 101 | + .map(([k, _]) => k); |
| 102 | + // This await would be dangerous without the lock held because it would allow interleaving |
| 103 | + // with the assignNodesToJob and performHealthCheck operations, both of which mutate |
| 104 | + // self.state.nodes. |
| 105 | + await unassignNodesForJob({ nodes: nodesToUnassign, jobName: input.jobName }); |
| 106 | + for (const node of nodesToUnassign) { |
| 107 | + this.state.nodes.set(node, null); |
| 108 | + } |
| 109 | + }); |
| 110 | + } |
| 111 | + |
| 112 | + async performHealthChecks(): Promise<void> { |
| 113 | + wf.log.info('performHealthChecks'); |
| 114 | + await this.nodesMutex.runExclusive(async () => { |
| 115 | + const badNodes = await findBadNodes({ nodesToCheck: Array.from(this.getAssignedNodes()) }); |
| 116 | + for (const node of badNodes) { |
| 117 | + this.state.nodes.set(node, 'BAD!'); |
| 118 | + } |
| 119 | + }); |
| 120 | + } |
| 121 | + |
| 122 | + getState(): ClusterManagerState { |
| 123 | + return { |
| 124 | + clusterStarted: this.state.clusterStarted, |
| 125 | + clusterShutdown: this.state.clusterShutdown, |
| 126 | + nodes: this.state.nodes, |
| 127 | + maxAssignedNodes: this.state.maxAssignedNodes, |
| 128 | + }; |
| 129 | + } |
| 130 | + |
| 131 | + getStateSummary(): ClusterManagerStateSummary { |
| 132 | + return { |
| 133 | + maxAssignedNodes: this.state.maxAssignedNodes, |
| 134 | + assignedNodes: this.getAssignedNodes().size, |
| 135 | + badNodes: this.getBadNodes().size, |
| 136 | + }; |
| 137 | + } |
| 138 | + |
| 139 | + getUnassignedNodes(): Set<string> { |
| 140 | + return new Set(Array.from(this.state.nodes.keys()).filter((key) => this.state.nodes.get(key) === null)); |
| 141 | + } |
| 142 | + |
| 143 | + getBadNodes(): Set<string> { |
| 144 | + return new Set(Array.from(this.state.nodes.keys()).filter((key) => this.state.nodes.get(key) === 'BAD!')); |
| 145 | + } |
| 146 | + |
| 147 | + getAssignedNodes(jobName?: string): Set<string> { |
| 148 | + return new Set( |
| 149 | + Array.from(this.state.nodes.keys()).filter((key) => { |
| 150 | + const value = this.state.nodes.get(key); |
| 151 | + return jobName ? value === jobName : value !== null && value !== 'BAD!'; |
| 152 | + }) |
| 153 | + ); |
| 154 | + } |
| 155 | +} |
0 commit comments