|
| 1 | +import json |
| 2 | +import subprocess |
| 3 | +from pathlib import Path |
| 4 | +from typing import Annotated |
| 5 | + |
1 | 6 | import typer |
| 7 | + |
2 | 8 | from pg2_benchmark.cli.dataset import dataset_app |
3 | 9 | from pg2_benchmark.cli.metric import metric_app |
4 | 10 | from pg2_benchmark.cli.sagemaker import sagemaker_app |
| 11 | +from pg2_benchmark.model_card import ModelCard |
| 12 | + |
| 13 | + |
| 14 | +class ModelPath: |
| 15 | + ROOT_PATH = Path("models") |
| 16 | + SRC_PATH = Path("src") |
| 17 | + PACKAGE_PREFIX = "pg2_model" |
| 18 | + MODEL_CARD_PATH = Path("README.md") |
| 19 | + MAIN_PY_PATH = Path("__main__.py") |
| 20 | + COMMAND_NAME = "train" |
| 21 | + COMMAND_PARAMS = ["dataset_file", "model_card_file"] |
| 22 | + |
5 | 23 |
|
6 | 24 | app = typer.Typer( |
7 | 25 | name="benchmark", |
|
15 | 33 |
|
16 | 34 |
|
17 | 35 | @app.command() |
18 | | -def ping(): |
19 | | - typer.echo("pong") |
| 36 | +def validate( |
| 37 | + model_name: Annotated[ |
| 38 | + str, typer.Argument(help="The model name listed in the `models` folder") |
| 39 | + ], |
| 40 | +): |
| 41 | + model_card_path = ModelPath.ROOT_PATH / model_name / ModelPath.MODEL_CARD_PATH |
| 42 | + |
| 43 | + if not model_card_path.exists(): |
| 44 | + typer.echo( |
| 45 | + f"❌ Model {model_name} does not have a model card at {model_card_path}" |
| 46 | + ) |
| 47 | + raise typer.Exit(1) |
| 48 | + |
| 49 | + try: |
| 50 | + model_card = ModelCard.from_path(model_card_path) |
| 51 | + typer.echo( |
| 52 | + f"✅ Loaded {model_card.name} with hyper parameters {model_card.hyper_params}." |
| 53 | + ) |
| 54 | + |
| 55 | + except Exception as e: |
| 56 | + typer.echo(f"❌ Error loading model card from {model_card_path}: {e}") |
| 57 | + raise typer.Exit(1) |
| 58 | + |
| 59 | + main_py_path = ( |
| 60 | + ModelPath.ROOT_PATH |
| 61 | + / model_name |
| 62 | + / ModelPath.SRC_PATH |
| 63 | + / f"{ModelPath.PACKAGE_PREFIX}_{model_name}" |
| 64 | + / ModelPath.MAIN_PY_PATH |
| 65 | + ) |
| 66 | + |
| 67 | + if not main_py_path.exists(): |
| 68 | + typer.echo( |
| 69 | + f"❌ Model {model_name} does not have a {ModelPath.MAIN_PY_PATH} file at {main_py_path}" |
| 70 | + ) |
| 71 | + raise typer.Exit(1) |
| 72 | + |
| 73 | + try: |
| 74 | + result = subprocess.run( |
| 75 | + [ |
| 76 | + "uv", |
| 77 | + "run", |
| 78 | + "--active", |
| 79 | + "python", |
| 80 | + "-c", |
| 81 | + f""" |
| 82 | +import importlib.util |
| 83 | +import inspect |
| 84 | +import json |
| 85 | +import sys |
| 86 | +
|
| 87 | +try: |
| 88 | + spec = importlib.util.spec_from_file_location('{ModelPath.PACKAGE_PREFIX}_{model_name}.__main__', '{ModelPath.PACKAGE_PREFIX}_{model_name}/__main__.py') |
| 89 | + module = importlib.util.module_from_spec(spec) |
| 90 | + spec.loader.exec_module(module) |
| 91 | +
|
| 92 | + app = getattr(module, "app") |
| 93 | + |
| 94 | + entrypoint_command_found = False |
| 95 | + entrypoint_params_found = False |
| 96 | +
|
| 97 | + for command in app.registered_commands: |
| 98 | + if '{ModelPath.COMMAND_NAME}' == command.callback.__name__: |
| 99 | + entrypoint_command_found = True |
| 100 | +
|
| 101 | + sig = inspect.signature(command.callback) |
| 102 | + |
| 103 | + if {ModelPath.COMMAND_PARAMS} == list(sig.parameters.keys()): |
| 104 | + entrypoint_params_found = True |
| 105 | + |
| 106 | + break |
| 107 | +
|
| 108 | + validation_result = {{ |
| 109 | + 'success': True, |
| 110 | + 'entrypoint_command_found': entrypoint_command_found, |
| 111 | + 'entrypoint_params_found': entrypoint_params_found, |
| 112 | + 'module_loaded': True |
| 113 | + }} |
| 114 | +
|
| 115 | + print(json.dumps(validation_result)) |
| 116 | + |
| 117 | +except Exception as e: |
| 118 | + error_result = {{ |
| 119 | + 'success': False, |
| 120 | + 'entrypoint_command_found': False, |
| 121 | + 'entrypoint_params_found': False, |
| 122 | + 'module_loaded': False, |
| 123 | + 'error': str(e) |
| 124 | + }} |
| 125 | + print(json.dumps(error_result)) |
| 126 | + sys.exit(1) |
| 127 | + """, |
| 128 | + ], |
| 129 | + cwd=ModelPath.ROOT_PATH / model_name / ModelPath.SRC_PATH, |
| 130 | + capture_output=True, |
| 131 | + text=True, |
| 132 | + ) |
| 133 | + |
| 134 | + if result.returncode == 0: |
| 135 | + validation_data = json.loads(result.stdout.strip()) |
| 136 | + |
| 137 | + if not validation_data["entrypoint_command_found"]: |
| 138 | + typer.echo( |
| 139 | + f"❌ Model {model_name} does not have a '{ModelPath.COMMAND_NAME}' command" |
| 140 | + ) |
| 141 | + raise typer.Exit(1) |
| 142 | + |
| 143 | + if not validation_data["entrypoint_params_found"]: |
| 144 | + typer.echo( |
| 145 | + f"❌ Model {model_name}'s '{ModelPath.COMMAND_NAME}' command does not have the required params: {ModelPath.COMMAND_PARAMS}" |
| 146 | + ) |
| 147 | + raise typer.Exit(1) |
| 148 | + |
| 149 | + typer.echo( |
| 150 | + f"✅ Model {model_name} has a valid '{ModelPath.COMMAND_NAME}' entrypoint with required params: {ModelPath.COMMAND_PARAMS}" |
| 151 | + ) |
| 152 | + else: |
| 153 | + typer.echo(f"❌ Error loading module {main_py_path}: {result.stderr}") |
| 154 | + raise typer.Exit(1) |
| 155 | + |
| 156 | + except Exception as e: |
| 157 | + typer.echo(f"❌ Error running validation subprocess: {e}") |
| 158 | + raise typer.Exit(1) |
20 | 159 |
|
21 | 160 |
|
22 | 161 | if __name__ == "__main__": |
|
0 commit comments