diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..146fc71 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,21 @@ +# EditorConfig is awesome: https://EditorConfig.org +# top-most EditorConfig file +root = true + +# Unix-style newlines with a newline ending every file + +[*] +# Unix-style newlines +end_of_line = lf +# Always end with an empty new line +insert_final_newline = true +# Set default charset to utf-8 +charset = utf-8 +# Indent with 2 spaces +indent_style = space +indent_size = 2 + +# 4 space indentation +[*.py] +indent_style = space +indent_size = 4 diff --git a/.gitignore b/.gitignore index 2fa7ce7..337c1bc 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,7 @@ config.ini + +# ignore venv +**/.venv + +# ignore python cache +**/__pycache__ diff --git a/Dockerfile b/Dockerfile index ae5e67f..1d16cfd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM python:alpine WORKDIR /usr/src/app RUN apk update -RUN apk add git rsync +RUN apk add git rsync terraform RUN apk add docker COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt diff --git a/config.ini b/config.ini index 6d35c62..09f7e8a 100644 --- a/config.ini +++ b/config.ini @@ -2,6 +2,8 @@ # ALLOWED_IP_RANGE = 192.168.0.0/24 # DEBUG = true # GIT_SSL_NO_VERIFY = true +# GIT_SSH_NO_VERIFY = true +# GIT_PROTOCOL = http # LISTEN_PORT = 1706 [rsync] diff --git a/requirements.txt b/requirements.txt index 84baa64..1216b2a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1 @@ -Flask>=2.1.0 -waitress>=2.1.0 +quart==0.18.3 diff --git a/runner/__init__.py b/runner/__init__.py new file mode 100644 index 0000000..5cb698f --- /dev/null +++ b/runner/__init__.py @@ -0,0 +1,123 @@ +import logging +from ipaddress import ip_address, ip_network +from os import access, environ, X_OK +from shutil import which + +from quart import Quart, request, jsonify + + +def create_app(config): + print("Tea Runner") + # Configure loglevel + if config.getboolean("runner", "DEBUG", fallback="False") == True: + logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.DEBUG) + logging.info("Debug logging is on") + else: + logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.INFO) + + app = Quart(__name__) + + # Configure Quart + app.runner_config = config + app.git_protocol = app.runner_config.get("runner", "GIT_PROTOCOL", fallback="http") + + # Check presence of external programs + app.docker = which("docker") + try: + access(app.docker, X_OK) + except: + logging.error("docker binary not found or not executable") + exit(1) + + app.git = which("git") + try: + access(app.git, X_OK) + except: + logging.error("git binary not found or not executable") + exit(1) + + app.rsync = which("rsync") + try: + access(app.rsync, X_OK) + except: + logging.error("rsync binary not found or not executable") + exit(1) + + app.tf_bin = which("terraform") + try: + access(app.tf_bin, X_OK) + except: + logging.error("terraform binary not found or not executable") + exit(1) + + # Set environment variables + if ( + app.runner_config.getboolean("runner", "GIT_SSL_NO_VERIFY", fallback="False") + == True + ): + environ["GIT_SSL_NO_VERIFY"] = "true" + if ( + app.runner_config.getboolean("runner", "GIT_SSH_NO_VERIFY", fallback="False") + == True + ): + environ[ + "GIT_SSH_COMMAND" + ] = "ssh -o UserKnownHostsFile=test -o StrictHostKeyChecking=no" + + # Log some informations + logging.info("git protocol is " + app.git_protocol) + logging.info( + "Limiting requests to: " + + app.runner_config.get("runner", "ALLOWED_IP_RANGE", fallback="") + ) + + @app.before_request + async def check_authorized(): + """ + Only respond to requests from ALLOWED_IP_RANGE if it's configured in config.ini + """ + if app.runner_config.has_option("runner", "ALLOWED_IP_RANGE"): + allowed_ip_range = ip_network( + app.runner_config["runner"]["ALLOWED_IP_RANGE"] + ) + requesting_ip = ip_address(request.remote_addr) + if requesting_ip not in allowed_ip_range: + logging.info( + "Dropping request from unauthorized host " + request.remote_addr + ) + return jsonify(status="forbidden"), 403 + else: + logging.info("Request from " + request.remote_addr) + + @app.before_request + async def check_media_type(): + """ + Only respond requests with Content-Type header of application/json + """ + if ( + not request.headers.get("Content-Type") + .lower() + .startswith("application/json") + ): + logging.error( + '"Content-Type: application/json" header missing from request made by ' + + request.remote_addr + ) + return jsonify(status="unsupported media type"), 415 + + @app.route("/test", methods=["POST"]) + async def test(): + logging.debug("Content-Type: " + request.headers.get("Content-Type")) + logging.debug(await request.get_json(force=True)) + return jsonify(status="success", sender=request.remote_addr) + + # Register Blueprints + from runner.docker import docker as docker_bp + from runner.rsync import rsync as rsync_bp + from runner.terraform import terraform as terraform_bp + + app.register_blueprint(docker_bp, url_prefix="/docker") + app.register_blueprint(rsync_bp) + app.register_blueprint(terraform_bp, url_prefix="/terraform") + + return app diff --git a/runner/docker.py b/runner/docker.py new file mode 100644 index 0000000..29239e0 --- /dev/null +++ b/runner/docker.py @@ -0,0 +1,38 @@ +import logging +from os import chdir, getcwd +from subprocess import run, DEVNULL +from tempfile import TemporaryDirectory + +from quart import Blueprint, current_app, jsonify, request + +import runner.utils + +docker = Blueprint("docker", __name__) + + +@docker.route("/build", methods=["POST"]) +async def docker_build(): + body = await request.get_json() + + with TemporaryDirectory() as temp_dir: + current_dir = getcwd() + if runner.utils.git_clone( + body["repository"]["clone_url"] + if current_app.git_protocol == "http" + else body["repository"]["ssh_url"], + temp_dir, + ): + logging.info("docker build") + chdir(temp_dir) + result = run( + [current_app.docker, "build", "-t", body["repository"]["name"], "."], + stdout=None if logging.root.level == logging.DEBUG else DEVNULL, + stderr=None if logging.root.level == logging.DEBUG else DEVNULL, + ) + chdir(current_dir) + if result.returncode != 0: + return jsonify(status="docker build failed"), 500 + else: + return jsonify(status="git clone failed"), 500 + + return jsonify(status="success") diff --git a/runner/rsync.py b/runner/rsync.py new file mode 100644 index 0000000..e657da5 --- /dev/null +++ b/runner/rsync.py @@ -0,0 +1,60 @@ +import logging +from os import chdir, getcwd, path +from subprocess import run, DEVNULL +from tempfile import TemporaryDirectory + +from quart import Blueprint, current_app, jsonify, request +from werkzeug import utils + +import runner.utils + +rsync = Blueprint("rsync", __name__) + + +@rsync.route("/rsync", methods=["POST"]) +async def route_rsync(): + body = await request.get_json() + dest = request.args.get("dest") or body["repository"]["name"] + rsync_root = current_app.runner_config.get("rsync", "RSYNC_ROOT", fallback="") + if rsync_root: + dest = path.join(rsync_root, utils.secure_filename(dest)) + logging.debug("rsync dest path updated to " + dest) + + with TemporaryDirectory() as temp_dir: + current_dir = getcwd() + if runner.utils.git_clone( + body["repository"]["clone_url"] + if current_app.git_protocol == "http" + else body["repository"]["ssh_url"], + temp_dir, + ): + logging.info("rsync " + body["repository"]["name"] + " to " + dest) + chdir(temp_dir) + if current_app.runner_config.get("rsync", "DELETE", fallback=""): + result = run( + [ + current_app.rsync, + "-r", + "--exclude=.git", + "--delete-during" + if current_app.runner_config.get("rsync", "DELETE", fallback="") + else "", + ".", + dest, + ], + stdout=None if logging.root.level == logging.DEBUG else DEVNULL, + stderr=None if logging.root.level == logging.DEBUG else DEVNULL, + ) + else: + result = run( + [current_app.rsync, "-r", "--exclude=.git", ".", dest], + stdout=None if logging.root.level == logging.DEBUG else DEVNULL, + stderr=None if logging.root.level == logging.DEBUG else DEVNULL, + ) + chdir(current_dir) + if result.returncode != 0: + return jsonify(status="rsync failed"), 500 + else: + return jsonify(status="git clone failed"), 500 + + return jsonify(status="success") diff --git a/runner/terraform.py b/runner/terraform.py new file mode 100644 index 0000000..daafbaa --- /dev/null +++ b/runner/terraform.py @@ -0,0 +1,81 @@ +import logging +from os import chdir, getcwd +from subprocess import run, DEVNULL +from tempfile import TemporaryDirectory + +from quart import Blueprint, current_app, jsonify, request + +import runner.utils + +terraform = Blueprint("terraform", __name__) + + +@terraform.route("/plan", methods=["POST"]) +async def terraform_plan(): + body = await request.get_json() + + with TemporaryDirectory() as temp_dir: + current_dir = getcwd() + if runner.utils.git_clone( + body["repository"]["clone_url"] + if current_app.git_protocol == "http" + else body["repository"]["ssh_url"], + temp_dir, + ): + logging.info("terraform init") + chdir(temp_dir) + result = run( + [current_app.tf_bin, "init", "-no-color"], + stdout=None if logging.root.level == logging.DEBUG else DEVNULL, + stderr=None if logging.root.level == logging.DEBUG else DEVNULL, + ) + if result.returncode != 0: + chdir(current_dir) + return jsonify(status="terraform init failed"), 500 + result = run( + [current_app.tf_bin, "plan", "-no-color"], stdout=None, stderr=None + ) + if result.returncode != 0: + chdir(current_dir) + return jsonify(status="terraform plan failed"), 500 + else: + chdir(current_dir) + return jsonify(status="git clone failed"), 500 + chdir(current_dir) + return jsonify(status="success") + + +@terraform.route("/apply", methods=["POST"]) +def terraform_apply(): + body = request.get_json() + with TemporaryDirectory() as temp_dir: + current_dir = getcwd() + if runner.utils.git_clone( + body["repository"]["clone_url"] + if current_app.git_protocol == "http" + else body["repository"]["ssh_url"], + temp_dir, + ): + logging.info("terraform init") + chdir(temp_dir) + result = run( + [current_app.tf_bin, "init", "-no-color"], + stdout=None if logging.root.level == logging.DEBUG else DEVNULL, + stderr=None if logging.root.level == logging.DEBUG else DEVNULL, + ) + if result.returncode != 0: + chdir(current_dir) + return jsonify(status="terraform init failed"), 500 + result = run( + [current_app.tf_bin, "apply", "-auto-approve", "-no-color"], + stdout=None if logging.root.level == logging.DEBUG else DEVNULL, + stderr=None if logging.root.level == logging.DEBUG else DEVNULL, + ) + if result.returncode != 0: + chdir(current_dir) + return jsonify(status="terraform apply failed"), 500 + else: + chdir(current_dir) + return jsonify(status="git clone failed"), 500 + chdir(current_dir) + return jsonify(status="success") diff --git a/runner/utils.py b/runner/utils.py new file mode 100644 index 0000000..631bdca --- /dev/null +++ b/runner/utils.py @@ -0,0 +1,29 @@ +import logging +from os import chdir, getcwd +from subprocess import run, DEVNULL + +from quart import current_app + + +def git_clone(src_url, dest_dir): + """ + Clone a remote git repository into a local directory. + + Args: + src_url (string): Url used to clone the repo. + dest_dir (string): Path to the local directory. + + Returns: + (boolean): True if command returns success. + """ + + logging.info("git clone " + src_url) + current_dir = getcwd() + chdir(dest_dir) + clone_result = run( + [current_app.git, "clone", src_url, "."], + stdout=None if logging.root.level == logging.DEBUG else DEVNULL, + stderr=None if logging.root.level == logging.DEBUG else DEVNULL, + ) + chdir(current_dir) + return clone_result.returncode == 0 diff --git a/tea_runner.py b/tea_runner.py index cc24979..6f05f45 100644 --- a/tea_runner.py +++ b/tea_runner.py @@ -11,181 +11,45 @@ [runner] ALLOWED_IP_RANGE=xxx.xxx.xxx.xxx/mm # Only respond to requests made from this range of IP addresses. Eg. 192.168.1.0/24 + GIT_PROTOCOL= + # Choose the protocol to use when cloning repositories. Default to http GIT_SSL_NO_VERIFY=true # Ignore certificate host verification errors. Useful for self-signed certs. + GIT_SSH_NO_VERIFY=true + # Ignore certificate host verification errors. LISTEN_IP=xxx.xxx.xxx.xxx # IP address for incoming requests. Defaults to 0.0.0.0 (Any). LISTEN_PORT=xxxx # TCP port number used for incoming requests. Defaults to 1706. """ -from ipaddress import ip_address, ip_network -from os import path -from subprocess import run, DEVNULL -from sys import exit -from os import access, X_OK, chdir, environ, path -from tempfile import TemporaryDirectory -from waitress import serve -from werkzeug import utils -from flask import Flask, request, jsonify -from configparser import ConfigParser +import asyncio from argparse import ArgumentParser -import logging +from configparser import ConfigParser -GIT_BIN = '/usr/bin/git' -RSYNC_BIN = '/usr/bin/rsync' -DOCKER_BIN = '/usr/bin/docker' +from hypercorn.asyncio import Config, serve -print("Tea Runner") +import runner # Debug is a command-line option, but most configuration comes from config.ini arg_parser = ArgumentParser() -arg_parser.add_argument('-d', '--debug', action='store_true', - help='display debugging output while running') +arg_parser.add_argument( + "-d", "--debug", action="store_true", help="display debugging output while running" +) args = arg_parser.parse_args() -config = ConfigParser() -config.read('config.ini') - -if args.debug: - config.set('runner', 'DEBUG', "true") - -if config.getboolean('runner', 'DEBUG', fallback='False') == True: - logging.basicConfig(format='%(levelname)s: %(message)s', - level=logging.DEBUG) - logging.info('Debug logging is on') -else: - logging.basicConfig( - format='%(levelname)s: %(message)s', level=logging.INFO) - -if not access(GIT_BIN, X_OK): - logging.error("git binary not found or not executable") - exit(1) -if not access(RSYNC_BIN, X_OK): - logging.error("rsync binary not found or not executable") - exit(1) -if not access(DOCKER_BIN, X_OK): - logging.error("docker binary not found or not executable") - exit(1) - - -def git_clone(src_url, dest_dir): - """ - Clone a remote git repository into a local directory. - - Args: - src_url (string): HTTP(S) url used to clone the repo. - dest_dir (string): Path to the local directory. - - Returns: - (boolean): True if command returns success. - """ - - logging.info('git clone ' + src_url) - if config.getboolean('runner', 'GIT_SSL_NO_VERIFY', fallback='False') == True: - environ['GIT_SSL_NO_VERIFY'] = 'true' - chdir(dest_dir) - clone_result = run([GIT_BIN, 'clone', src_url, '.'], - stdout=None if args.debug else DEVNULL, stderr=None if args.debug else DEVNULL) - return clone_result.returncode == 0 - - -app = Flask(__name__) - +quart_config = ConfigParser() +quart_config.read("config.ini") +hypercorn_config = Config() -@app.before_request -def check_authorized(): - """ - Only respond to requests from ALLOWED_IP_RANGE if it's configured in config.ini - """ - if config.has_option('runner', 'ALLOWED_IP_RANGE'): - allowed_ip_range = ip_network(config['runner']['ALLOWED_IP_RANGE']) - requesting_ip = ip_address(request.remote_addr) - if requesting_ip not in allowed_ip_range: - logging.info( - 'Dropping request from unauthorized host ' + request.remote_addr) - return jsonify(status='forbidden'), 403 - else: - logging.info('Request from ' + request.remote_addr) - - -@app.before_request -def check_media_type(): - """ - Only respond requests with Content-Type header of application/json - """ - if not request.headers.get('Content-Type').lower().startswith('application/json'): - logging.error( - '"Content-Type: application/json" header missing from request made by ' + request.remote_addr) - return jsonify(status='unsupported media type'), 415 - - -@app.route('/test', methods=['POST']) -def test(): - logging.debug('Content-Type: ' + request.headers.get('Content-Type')) - logging.debug(request.get_json(force=True)) - return jsonify(status='success', sender=request.remote_addr) - - -@app.route('/rsync', methods=['POST']) -def rsync(): - body = request.get_json() - dest = request.args.get('dest') or body['repository']['name'] - rsync_root = config.get('rsync', 'RSYNC_ROOT', fallback='') - if rsync_root: - dest = path.join(rsync_root, utils.secure_filename(dest)) - logging.debug('rsync dest path updated to ' + dest) - - with TemporaryDirectory() as temp_dir: - if git_clone(body['repository']['clone_url'], temp_dir): - logging.info('rsync ' + body['repository']['name'] + ' to ' + dest) - chdir(temp_dir) - if config.get('rsync', 'DELETE', fallback=''): - result = run([RSYNC_BIN, '-r', - '--exclude=.git', - '--delete-during' if config.get( - 'rsync', 'DELETE', fallback='') else '', - '.', - dest], - stdout=None if args.debug else DEVNULL, - stderr=None if args.debug else DEVNULL - ) - else: - result = run([RSYNC_BIN, '-r', - '--exclude=.git', - '.', - dest], - stdout=None if args.debug else DEVNULL, - stderr=None if args.debug else DEVNULL - ) - if result.returncode != 0: - return jsonify(status='rsync failed'), 500 - else: - return jsonify(status='git clone failed'), 500 - - return jsonify(status='success') - - -@app.route('/docker/build', methods=['POST']) -def docker_build(): - body = request.get_json() - - with TemporaryDirectory() as temp_dir: - if git_clone(body['repository']['clone_url'], temp_dir): - logging.info('docker build') - chdir(temp_dir) - result = run([DOCKER_BIN, 'build', '-t', body['repository']['name'], '.'], - stdout=None if args.debug else DEVNULL, stderr=None if args.debug else DEVNULL) - if result.returncode != 0: - return jsonify(status='docker build failed'), 500 - else: - return jsonify(status='git clone failed'), 500 - - return jsonify(status='success') +hypercorn_config.bind = ( + quart_config.get("runner", "LISTEN_IP", fallback="0.0.0.0") + + ":" + + str(quart_config.getint("runner", "LISTEN_PORT", fallback=1706)) +) +if args.debug: + quart_config.set("runner", "DEBUG", "true") + hypercorn_config.loglevel = "debug" -if __name__ == '__main__': - logging.info('Limiting requests to: ' + config.get('runner', - 'ALLOWED_IP_RANGE', fallback='')) - serve(app, host=config.get('runner', 'LISTEN_IP', fallback='0.0.0.0'), - port=config.getint('runner', 'LISTEN_PORT', fallback=1706)) +asyncio.run(serve(runner.create_app(quart_config), hypercorn_config))