|
| 1 | +#!/usr/bin/env python3 |
| 2 | + |
| 3 | +import argparse |
| 4 | +import pathlib |
| 5 | +import os |
| 6 | +import tempfile |
| 7 | +import json |
| 8 | +import sqlite3 |
| 9 | +import zipfile |
| 10 | +import atexit |
| 11 | +import shutil |
| 12 | +import subprocess |
| 13 | +import sys |
| 14 | +import selectors |
| 15 | +import logging |
| 16 | +import platform |
| 17 | + |
| 18 | +logging.basicConfig( |
| 19 | + level=logging.INFO, |
| 20 | + stream=sys.stdout |
| 21 | +) |
| 22 | + |
| 23 | +def find_default_file(source, ext): |
| 24 | + if os.path.isfile(source): |
| 25 | + return source if source.name.endswith(ext) else None |
| 26 | + files = [x for x in os.listdir(source) if x.endswith(ext)] |
| 27 | + if len(files) == 1: |
| 28 | + return os.path.join(source, files[0]) |
| 29 | + if len(files) > 1: |
| 30 | + raise Exception(f"More than one {ext} file found, can not continue") |
| 31 | + return None |
| 32 | + |
| 33 | +def get_metadata_value(source_dir): |
| 34 | + file_name = os.path.join(source_dir, 'model', 'metadata.json') |
| 35 | + try: |
| 36 | + with open(file_name) as file_handle: |
| 37 | + return json.loads(file_handle.read()) |
| 38 | + except IOError: |
| 39 | + return None |
| 40 | + |
| 41 | +def extract_zip(mda_file): |
| 42 | + temp_dir = tempfile.TemporaryDirectory(prefix='mendix-docker-buildpack') |
| 43 | + with zipfile.ZipFile(mda_file) as zip_file: |
| 44 | + zip_file.extractall(temp_dir.name) |
| 45 | + return temp_dir |
| 46 | + |
| 47 | +BUILDER_PROCESS = None |
| 48 | +def stop_processes(): |
| 49 | + if BUILDER_PROCESS is not None: |
| 50 | + proc = BUILDER_PROCESS |
| 51 | + proc.terminate() |
| 52 | + proc.communicate() |
| 53 | + proc.wait() |
| 54 | + |
| 55 | +def container_call(args): |
| 56 | + build_executables = ['podman', 'docker'] |
| 57 | + build_executable = None |
| 58 | + logger_stdout = None |
| 59 | + logger_stderr = None |
| 60 | + for builder in build_executables: |
| 61 | + build_executable = shutil.which(builder) |
| 62 | + if build_executable is not None: |
| 63 | + logger_stderr = logging.getLogger(builder + '-stderr') |
| 64 | + logger_stdout = logging.getLogger(builder + '-stdout') |
| 65 | + break |
| 66 | + if build_executable is None: |
| 67 | + raise Exception('Cannot find Podman or Docker executable') |
| 68 | + proc = subprocess.Popen([build_executable] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) |
| 69 | + BUILDER_PROCESS = proc |
| 70 | + |
| 71 | + sel = selectors.DefaultSelector() |
| 72 | + sel.register(proc.stdout, selectors.EVENT_READ) |
| 73 | + sel.register(proc.stderr, selectors.EVENT_READ) |
| 74 | + |
| 75 | + last_line_stdout = None |
| 76 | + last_line_stderr = None |
| 77 | + stdout_open, stderr_open = True, True |
| 78 | + while stdout_open or stderr_open: |
| 79 | + for key, _ in sel.select(): |
| 80 | + data = key.fileobj.readline() |
| 81 | + if data == '': |
| 82 | + if key.fileobj is proc.stdout: |
| 83 | + stdout_open = False |
| 84 | + elif key.fileobj is proc.stderr: |
| 85 | + stderr_open = False |
| 86 | + continue |
| 87 | + data = data.rstrip() |
| 88 | + if key.fileobj is proc.stdout: |
| 89 | + last_line_stdout = data |
| 90 | + logger_stdout.info(data) |
| 91 | + elif key.fileobj is proc.stderr: |
| 92 | + last_line_stderr = data |
| 93 | + # stderr is mostly used for progress notifications, not errors |
| 94 | + logger_stderr.info(data) |
| 95 | + |
| 96 | + sel.close() |
| 97 | + BUILDER_PROCESS = None |
| 98 | + if proc.wait() != 0: |
| 99 | + raise Exception(f"Builder returned with error: {last_line_stderr}") |
| 100 | + return last_line_stdout |
| 101 | + |
| 102 | +def pull_image(image_url): |
| 103 | + try: |
| 104 | + container_call(['image', 'pull', image_url]) |
| 105 | + return image_url |
| 106 | + except: |
| 107 | + return None |
| 108 | + |
| 109 | +def delete_container(container_id): |
| 110 | + try: |
| 111 | + container_call(['container', 'rm', '--force', container_id]) |
| 112 | + except Exception as e: |
| 113 | + logging.warning('Failed to delete container {}: {}'.format(container_id, e)) |
| 114 | + |
| 115 | +def build_mpr_builder(mx_version, dotnet, artifacts_repository=None): |
| 116 | + builder_image_tag = f"mxbuild-{mx_version}-{dotnet}-{platform.machine()}" |
| 117 | + builder_image_url = None |
| 118 | + if artifacts_repository is not None: |
| 119 | + builder_image_url = f"{artifacts_repository}:{builder_image_tag}" |
| 120 | + image_url = pull_image(builder_image_url) |
| 121 | + if image_url is not None: |
| 122 | + return image_url |
| 123 | + else: |
| 124 | + builder_image_url = f"mendix-buildpack:{builder_image_tag}" |
| 125 | + |
| 126 | + prefix = '' |
| 127 | + if platform.machine() == 'arm64' and dotnet == 'dotnet': |
| 128 | + prefix = 'arm64-' |
| 129 | + |
| 130 | + mxbuild_filename = f"{prefix}mxbuild-{mx_version}.tar.gz" |
| 131 | + mxbuild_url = f"https://download.mendix.com/runtimes/{mxbuild_filename}" |
| 132 | + |
| 133 | + build_args = ['--build-arg', f"MXBUILD_DOWNLOAD_URL={mxbuild_url}", |
| 134 | + '--file', os.path.join('mxbuild', f"{dotnet}.dockerfile"), |
| 135 | + '--tag', builder_image_url] |
| 136 | + |
| 137 | + container_call(['image', 'build'] + build_args + ['mxbuild']) |
| 138 | + if artifacts_repository is not None: |
| 139 | + try: |
| 140 | + container_call(['image', 'push', builder_image_url]) |
| 141 | + except Exception as e: |
| 142 | + logging.warning('Failed to push mxbuild into artifacts repository: {}; continuing with the build'.format(e)) |
| 143 | + return builder_image_url |
| 144 | + |
| 145 | +def get_git_commit(source_dir): |
| 146 | + git_head = os.path.join(source_dir, '.git', 'HEAD') |
| 147 | + if not os.path.isfile(git_head): |
| 148 | + raise Exception('Project source doesn\'t contain git metadata') |
| 149 | + with open(git_head) as git_head: |
| 150 | + git_head_line = git_head.readline().split() |
| 151 | + if len(git_head_line) == 1: |
| 152 | + # Detached commit |
| 153 | + return git_head_line[0] |
| 154 | + if len(git_head_line) > 2: |
| 155 | + raise Exception(f"Unsupported Git HEAD format {git_head_line}") |
| 156 | + git_branch = git_head_line[1].split('/') |
| 157 | + git_branch_file = os.path.join(*([source_dir, '.git'] + git_branch)) |
| 158 | + if not os.path.isfile(git_branch_file): |
| 159 | + raise Exception('Git branch file doesn\'t exist') |
| 160 | + with open(git_branch_file) as git_branch_file: |
| 161 | + return git_branch_file.readline() |
| 162 | + |
| 163 | + |
| 164 | +def build_mpr(source_dir, mpr_file, destination, artifacts_repository=None): |
| 165 | + cursor = sqlite3.connect(mpr_file).cursor() |
| 166 | + cursor.execute("SELECT _ProductVersion FROM _MetaData LIMIT 1") |
| 167 | + mx_version = cursor.fetchone()[0] |
| 168 | + mx_version_value = parse_version(mx_version) |
| 169 | + logging.debug('Detected Mendix version {}'.format('.'.join(map(str,mx_version_value)))) |
| 170 | + dotnet = 'dotnet' if mx_version_value >= (10, 0, 0, 0) else 'mono' |
| 171 | + builder_image = build_mpr_builder(mx_version, dotnet, artifacts_repository) |
| 172 | + model_version = None |
| 173 | + try: |
| 174 | + model_version = get_git_commit(source_dir) |
| 175 | + except Exception as e: |
| 176 | + model_version = 'unversioned' |
| 177 | + logging.warning('Cannot determine git commit ({}), will set model version to unversioned'.format(e)) |
| 178 | + container_id = container_call(['container', 'create', builder_image, os.path.basename(mpr_file), model_version]) |
| 179 | + atexit.register(delete_container, container_id) |
| 180 | + container_call(['container', 'cp', os.path.abspath(source_dir)+'/.', f"{container_id}:/workdir/project"]) |
| 181 | + build_result = container_call(['start', '--attach', '--interactive', container_id]) |
| 182 | + |
| 183 | + temp_dir = tempfile.TemporaryDirectory(prefix='mendix-docker-buildpack') |
| 184 | + container_call(['container', 'cp', f"{container_id}:/workdir/output.mda", temp_dir.name]) |
| 185 | + with zipfile.ZipFile(os.path.join(temp_dir.name, 'output.mda')) as zip_file: |
| 186 | + zip_file.extractall(destination) |
| 187 | + |
| 188 | +def parse_version(version): |
| 189 | + return tuple([ int(n) for n in version.split('.') ]) |
| 190 | + |
| 191 | +def prepare_destination(destination_path): |
| 192 | + with os.scandir(destination_path) as entries: |
| 193 | + for entry in entries: |
| 194 | + if entry.is_dir() and not entry.is_symlink(): |
| 195 | + shutil.rmtree(entry.path) |
| 196 | + else: |
| 197 | + os.remove(entry.path) |
| 198 | + project_path = os.path.join(destination_path, 'project') |
| 199 | + os.mkdir(project_path, 0o755) |
| 200 | + shutil.copytree('scripts', os.path.join(destination_path, 'scripts')) |
| 201 | + shutil.copyfile('Dockerfile', os.path.join(destination_path, 'Dockerfile')) |
| 202 | + return project_path |
| 203 | + |
| 204 | +def prepare_mda(source_path, destination_path, artifacts_repository=None): |
| 205 | + destination_path = prepare_destination(destination_path) |
| 206 | + mpk_file = find_default_file(source_path, '.mpk') |
| 207 | + extracted_dir = None |
| 208 | + if mpk_file is not None: |
| 209 | + extracted_dir = extract_zip(mpk_file) |
| 210 | + source_path = extracted_dir.name |
| 211 | + mpr_file = find_default_file(source_path, '.mpr') |
| 212 | + if mpr_file is not None: |
| 213 | + source_path = os.path.abspath(os.path.join(mpr_file, os.pardir)) |
| 214 | + return build_mpr(source_path, mpr_file, destination_path, artifacts_repository) |
| 215 | + mda_file = find_default_file(source_path, '.mda') |
| 216 | + if mda_file is not None: |
| 217 | + with zipfile.ZipFile(mda_file) as zip_file: |
| 218 | + zip_file.extractall(destination_path) |
| 219 | + elif os.path.isdir(source_path): |
| 220 | + shutil.copytree(source_path, destination_path, dirs_exist_ok=True) |
| 221 | + extracted_mda_file = get_metadata_value(destination_path) |
| 222 | + if extracted_mda_file is not None: |
| 223 | + return destination_path |
| 224 | + else: |
| 225 | + raise Exception('No supported files found in source path') |
| 226 | + |
| 227 | +def build_image(mda_dir): |
| 228 | + # TODO: build the full image, or just copy MDA into destination? |
| 229 | + mda_path = mda_dir.name if isinstance(mda_dir, tempfile.TemporaryDirectory) else mda_dir |
| 230 | + mda_metadata = get_metadata_value(mda_path) |
| 231 | + mx_version = mda_metadata['RuntimeVersion'] |
| 232 | + java_version = mda_metadata.get('JavaVersion', 11) |
| 233 | + logging.debug("Detected Mendix {} Java {}".format(mx_version, java_version)) |
| 234 | + |
| 235 | +if __name__ == '__main__': |
| 236 | + parser = argparse.ArgumentParser(description='Build a Mendix app') |
| 237 | + parser.add_argument('--source', metavar='source', required=True, nargs='?', type=pathlib.Path, help='Path to source Mendix app (MDA file, MPK file, MPR directory or extracted MDA directory)') |
| 238 | + parser.add_argument('--destination', metavar='destination', required=True, nargs='?', type=pathlib.Path, help='Destination for MDA') |
| 239 | + parser.add_argument('--artifacts-repository', required=False, nargs='?', metavar='artifacts_repository', type=str, help='Repository to use for caching build images') |
| 240 | + parser.add_argument('action', metavar='action', choices=['build-mda-dir'], help='Action to perform') |
| 241 | + |
| 242 | + args = parser.parse_args() |
| 243 | + |
| 244 | + atexit.register(stop_processes) |
| 245 | + try: |
| 246 | + prepare_mda(args.source, args.destination, args.artifacts_repository) |
| 247 | + except KeyboardInterrupt: |
| 248 | + stop_processes() |
| 249 | + raise |
| 250 | + # build_image(args.destination) |
0 commit comments