|
| 1 | +#!/usr/bin/env python3 |
| 2 | + |
| 3 | +# Copyright (c) 2024 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. |
| 4 | +# SPDX-License-Identifier: Apache-2.0 |
| 5 | + |
| 6 | +# This implements a very simple RPC server that should be running on the job container of the workflow pod, |
| 7 | +# and used by the k8s hook to execute steps in the workflow on the workflow pod. |
| 8 | + |
| 9 | +# It supports a running a single RPC call at a time, and will return an error if a new call is made while |
| 10 | +# another one is still running (which is a valid assumption, as the runner is expected to execute one step at a time). |
| 11 | + |
| 12 | + |
| 13 | +from concurrent.futures import ThreadPoolExecutor |
| 14 | +from dataclasses import dataclass |
| 15 | +import time |
| 16 | +from flask import Flask, jsonify, request |
| 17 | +from threading import Thread |
| 18 | +from waitress import serve |
| 19 | + |
| 20 | +import argparse |
| 21 | +import json |
| 22 | +import logging |
| 23 | +import os |
| 24 | +import signal |
| 25 | +import subprocess |
| 26 | + |
| 27 | +import logging |
| 28 | +import json_logging |
| 29 | + |
| 30 | +app = Flask(__name__) |
| 31 | +app.logger.setLevel(logging.DEBUG) |
| 32 | +json_logging.init_flask(enable_json=True) |
| 33 | +json_logging.init_request_instrument(app) |
| 34 | + |
| 35 | +@dataclass |
| 36 | +class Response: |
| 37 | + id: str |
| 38 | + status: str |
| 39 | + pid: int = None |
| 40 | + returncode: int = None |
| 41 | + error: str = None |
| 42 | + |
| 43 | +def readLines(path, fromLine, maxLines): |
| 44 | + try: |
| 45 | + with open(path, 'r') as f: |
| 46 | + return [x for i, x in enumerate(f) if i >= fromLine and x.endswith('\n') and i < fromLine + maxLines] |
| 47 | + except Exception as e: |
| 48 | + app.logger.warning(f"Error reading file {path}: {e}") |
| 49 | + return [] |
| 50 | + |
| 51 | +class State: |
| 52 | + def __init__(self): |
| 53 | + self.latest_id = None |
| 54 | + self.status = Response(id = "", status = "idle") |
| 55 | + self.worker = ThreadPoolExecutor(max_workers=1) |
| 56 | + self.future = None |
| 57 | + self.process = None |
| 58 | + self.out = None |
| 59 | + |
| 60 | + def __run(self, id, path): |
| 61 | + self.latest_id = id |
| 62 | + try: |
| 63 | + app.logger.debug(f"Running id {id}") |
| 64 | + logsfilename = f"/logs/{id}.out" |
| 65 | + self.out = open(logsfilename, "w") |
| 66 | + self.process = subprocess.Popen(['sh', '-e', path], start_new_session=True, stdout=self.out, stderr=self.out) |
| 67 | + app.logger.debug(f"Process for id {id} started with pid {self.process.pid}") |
| 68 | + self.status = Response( |
| 69 | + id = id, |
| 70 | + status = 'running', |
| 71 | + pid = self.process.pid |
| 72 | + ) |
| 73 | + self.process.wait() |
| 74 | + self.out.close() |
| 75 | + app.logger.debug(f"Process for id {id} finished (return code {self.process.returncode})") |
| 76 | + self.status = Response( |
| 77 | + id = id, |
| 78 | + status = 'completed', |
| 79 | + returncode = self.process.returncode, |
| 80 | + ) |
| 81 | + except Exception as e: |
| 82 | + app.logger.error(f"Error starting process: {e}") |
| 83 | + self.status = Response( |
| 84 | + id = id, |
| 85 | + status = 'failed', |
| 86 | + error = str(e), |
| 87 | + returncode = -1, |
| 88 | + ) |
| 89 | + |
| 90 | + |
| 91 | + def exec(self, id, path): |
| 92 | + if self.future and not self.future.done(): |
| 93 | + app.logger.error(f"A job is already running (ID {self.latest_id})") |
| 94 | + return Response( |
| 95 | + id = id, |
| 96 | + status = 'failed', |
| 97 | + error = f"A job is already running (ID {self.latest_id})", |
| 98 | + returncode = -1, |
| 99 | + ) |
| 100 | + |
| 101 | + app.logger.debug(f"Queueing job {id} with path {path}") |
| 102 | + self.status = Response(id = id, status = "pending") |
| 103 | + self.future = self.worker.submit(self.__run, id, path) |
| 104 | + return self.status |
| 105 | + |
| 106 | + def cancel(self): |
| 107 | + if not self.future: |
| 108 | + return Response( |
| 109 | + id = '', |
| 110 | + status = 'failed', |
| 111 | + error = 'No job has been started yet', |
| 112 | + ) |
| 113 | + elif self.future.done(): |
| 114 | + # The job is already done, no need to cancel |
| 115 | + return self.status |
| 116 | + else: |
| 117 | + app.logger.debug(f"Cancelling {self.latest_id} (pid {self.process.pid})") |
| 118 | + os.killpg(os.getpgid(self.process.pid), signal.SIGINT) |
| 119 | + |
| 120 | + return Response( |
| 121 | + id = self.latest_id, |
| 122 | + status = 'cancelling', |
| 123 | + pid = self.process.pid |
| 124 | + ) |
| 125 | + |
| 126 | +state = State() |
| 127 | + |
| 128 | +# Post a new job |
| 129 | +@app.route('/', methods=['POST']) |
| 130 | +def call(): |
| 131 | + data = json.loads(request.data) |
| 132 | + if 'id' not in data or 'path' not in data: |
| 133 | + return jsonify(Response( |
| 134 | + id = '', |
| 135 | + status = 'failed', |
| 136 | + error = 'Missing id or path in request', |
| 137 | + )) |
| 138 | + id = data['id'] |
| 139 | + path = data['path'] |
| 140 | + return jsonify(state.exec(id, path)) |
| 141 | + |
| 142 | +# Cancel the current job |
| 143 | +@app.route('/', methods=['DELETE']) |
| 144 | +def cancel(): |
| 145 | + return jsonify(state.cancel()) |
| 146 | + |
| 147 | +# Get the current status |
| 148 | +@app.route('/') |
| 149 | +def status(): |
| 150 | + app.logger.debug(f"Status: {state.status}") |
| 151 | + return jsonify(state.status) |
| 152 | + |
| 153 | +# Get the logs of a given job |
| 154 | +@app.route('/logs') |
| 155 | +def logs(): |
| 156 | + if 'id' not in request.args: |
| 157 | + return 'Missing id in request', 400 |
| 158 | + id = request.args.get('id') |
| 159 | + fromLine = int(request.args.get('fromLine', 0)) |
| 160 | + maxLines = int(request.args.get('maxLines', 1000)) |
| 161 | + path = f"/logs/{id}.out" |
| 162 | + return jsonify(readLines(path, fromLine, maxLines)) |
| 163 | + |
| 164 | + |
| 165 | +if __name__ == '__main__': |
| 166 | + |
| 167 | + parser = argparse.ArgumentParser() |
| 168 | + parser.add_argument('--dev', action='store_true', help='Run in Flask development mode') |
| 169 | + args = parser.parse_args() |
| 170 | + if args.dev: |
| 171 | + app.run(host='0.0.0.0', port=8080, debug=True) |
| 172 | + else: |
| 173 | + serve(app, host='0.0.0.0', port=8080, threads=1) |
| 174 | + |
0 commit comments