|
| 1 | +# MIT License |
| 2 | + |
| 3 | +# Copyright (c) 2024 The HuggingFace Team |
| 4 | + |
| 5 | +# Permission is hereby granted, free of charge, to any person obtaining a copy |
| 6 | +# of this software and associated documentation files (the "Software"), to deal |
| 7 | +# in the Software without restriction, including without limitation the rights |
| 8 | +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
| 9 | +# copies of the Software, and to permit persons to whom the Software is |
| 10 | +# furnished to do so, subject to the following conditions: |
| 11 | + |
| 12 | +# The above copyright notice and this permission notice shall be included in all |
| 13 | +# copies or substantial portions of the Software. |
| 14 | + |
| 15 | +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 16 | +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 17 | +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
| 18 | +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 19 | +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| 20 | +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 21 | +# SOFTWARE. |
| 22 | + |
| 23 | +""" |
| 24 | +Common CLI argument types for LightEval main files. |
| 25 | +This module exports pre-defined argument types to reduce redundancy across main_*.py files. |
| 26 | +""" |
| 27 | + |
| 28 | +from dataclasses import dataclass |
| 29 | +from typing import Any, Optional |
| 30 | + |
| 31 | +from typer import Argument, Option |
| 32 | +from typing_extensions import Annotated |
| 33 | + |
| 34 | + |
| 35 | +# Help panel names for consistent organization |
| 36 | +HELP_PANEL_NAME_1 = "Common Parameters" |
| 37 | +HELP_PANEL_NAME_2 = "Logging Parameters" |
| 38 | +HELP_PANEL_NAME_3 = "Debug Parameters" |
| 39 | +HELP_PANEL_NAME_4 = "Modeling Parameters" |
| 40 | + |
| 41 | + |
| 42 | +@dataclass |
| 43 | +class Arg: |
| 44 | + """Base class for CLI arguments with type and default value.""" |
| 45 | + |
| 46 | + type: Annotated |
| 47 | + default: Any |
| 48 | + |
| 49 | + |
| 50 | +# Common Parameters (HELP_PANEL_NAME_1) |
| 51 | +dataset_loading_processes = Arg( |
| 52 | + type=Annotated[ |
| 53 | + int, |
| 54 | + Option( |
| 55 | + help="Number of parallel processes to use for loading datasets. Higher values can speed up dataset loading but use more memory.", |
| 56 | + rich_help_panel=HELP_PANEL_NAME_1, |
| 57 | + ), |
| 58 | + ], |
| 59 | + default=1, |
| 60 | +) |
| 61 | + |
| 62 | +custom_tasks = Arg( |
| 63 | + type=Annotated[ |
| 64 | + Optional[str], |
| 65 | + Option( |
| 66 | + help="Path to a Python file containing custom task definitions. The file should define a TASKS_TABLE with LightevalTaskConfig objects.", |
| 67 | + rich_help_panel=HELP_PANEL_NAME_1, |
| 68 | + ), |
| 69 | + ], |
| 70 | + default=None, |
| 71 | +) |
| 72 | + |
| 73 | +num_fewshot_seeds = Arg( |
| 74 | + type=Annotated[ |
| 75 | + int, |
| 76 | + Option( |
| 77 | + help="Number of different random seeds to use for few-shot evaluation. Each seed will generate different few-shot examples, providing more robust evaluation.", |
| 78 | + rich_help_panel=HELP_PANEL_NAME_1, |
| 79 | + ), |
| 80 | + ], |
| 81 | + default=1, |
| 82 | +) |
| 83 | + |
| 84 | +load_responses_from_details_date_id = Arg( |
| 85 | + type=Annotated[ |
| 86 | + Optional[str], |
| 87 | + Option( |
| 88 | + help="Load previously generated model responses from a specific evaluation run instead of running the model. Use the timestamp/date_id from a previous run's details directory.", |
| 89 | + rich_help_panel=HELP_PANEL_NAME_1, |
| 90 | + ), |
| 91 | + ], |
| 92 | + default=None, |
| 93 | +) |
| 94 | + |
| 95 | +remove_reasoning_tags = Arg( |
| 96 | + type=Annotated[ |
| 97 | + bool, |
| 98 | + Option( |
| 99 | + help="Whether to remove reasoning tags from model responses before computing metrics.", |
| 100 | + rich_help_panel=HELP_PANEL_NAME_1, |
| 101 | + ), |
| 102 | + ], |
| 103 | + default=True, |
| 104 | +) |
| 105 | + |
| 106 | +reasoning_tags = Arg( |
| 107 | + type=Annotated[ |
| 108 | + str, |
| 109 | + Option( |
| 110 | + help="List of reasoning tag pairs to remove from responses, formatted as a Python list of tuples.", |
| 111 | + rich_help_panel=HELP_PANEL_NAME_1, |
| 112 | + ), |
| 113 | + ], |
| 114 | + default="[('<think>', '</think>')]", |
| 115 | +) |
| 116 | + |
| 117 | + |
| 118 | +# Logging Parameters (HELP_PANEL_NAME_2) |
| 119 | +output_dir = Arg( |
| 120 | + type=Annotated[ |
| 121 | + str, |
| 122 | + Option( |
| 123 | + help="Directory where evaluation results and details will be saved. Supports fsspec-compliant paths (local, s3, hf hub, etc.).", |
| 124 | + rich_help_panel=HELP_PANEL_NAME_2, |
| 125 | + ), |
| 126 | + ], |
| 127 | + default="results", |
| 128 | +) |
| 129 | + |
| 130 | +results_path_template = Arg( |
| 131 | + type=Annotated[ |
| 132 | + str | None, |
| 133 | + Option( |
| 134 | + help="Custom template for results file path. Available variables: {output_dir}, {org}, {model}. Example: '{output_dir}/experiments/{org}_{model}' creates results in a subdirectory.", |
| 135 | + rich_help_panel=HELP_PANEL_NAME_2, |
| 136 | + ), |
| 137 | + ], |
| 138 | + default=None, |
| 139 | +) |
| 140 | + |
| 141 | +push_to_hub = Arg( |
| 142 | + type=Annotated[ |
| 143 | + bool, |
| 144 | + Option( |
| 145 | + help="Whether to push evaluation results and details to the Hugging Face Hub. Requires --results-org to be set.", |
| 146 | + rich_help_panel=HELP_PANEL_NAME_2, |
| 147 | + ), |
| 148 | + ], |
| 149 | + default=False, |
| 150 | +) |
| 151 | + |
| 152 | +push_to_tensorboard = Arg( |
| 153 | + type=Annotated[ |
| 154 | + bool, |
| 155 | + Option( |
| 156 | + help="Whether to create and push TensorBoard logs to the Hugging Face Hub. Requires --results-org to be set.", |
| 157 | + rich_help_panel=HELP_PANEL_NAME_2, |
| 158 | + ), |
| 159 | + ], |
| 160 | + default=False, |
| 161 | +) |
| 162 | + |
| 163 | +public_run = Arg( |
| 164 | + type=Annotated[ |
| 165 | + bool, |
| 166 | + Option( |
| 167 | + help="Whether to make the uploaded results and details public on the Hugging Face Hub. If False, datasets will be private.", |
| 168 | + rich_help_panel=HELP_PANEL_NAME_2, |
| 169 | + ), |
| 170 | + ], |
| 171 | + default=False, |
| 172 | +) |
| 173 | + |
| 174 | +results_org = Arg( |
| 175 | + type=Annotated[ |
| 176 | + Optional[str], |
| 177 | + Option( |
| 178 | + help="Hugging Face organization where results will be pushed. Required when using --push-to-hub or --push-to-tensorboard.", |
| 179 | + rich_help_panel=HELP_PANEL_NAME_2, |
| 180 | + ), |
| 181 | + ], |
| 182 | + default=None, |
| 183 | +) |
| 184 | + |
| 185 | +save_details = Arg( |
| 186 | + type=Annotated[ |
| 187 | + bool, |
| 188 | + Option( |
| 189 | + help="Whether to save detailed per-sample results including model inputs, outputs, and metrics. Useful for analysis and debugging.", |
| 190 | + rich_help_panel=HELP_PANEL_NAME_2, |
| 191 | + ), |
| 192 | + ], |
| 193 | + default=False, |
| 194 | +) |
| 195 | + |
| 196 | +wandb = Arg( |
| 197 | + type=Annotated[ |
| 198 | + bool, |
| 199 | + Option( |
| 200 | + help="Whether to log results to Weights & Biases (wandb) or Trackio. Configure with environment variables: WANDB_PROJECT, WANDB_SPACE_ID, etc. See wandb docs for full configuration options.", |
| 201 | + rich_help_panel=HELP_PANEL_NAME_2, |
| 202 | + ), |
| 203 | + ], |
| 204 | + default=False, |
| 205 | +) |
| 206 | + |
| 207 | + |
| 208 | +# Debug Parameters (HELP_PANEL_NAME_3) |
| 209 | +max_samples = Arg( |
| 210 | + type=Annotated[ |
| 211 | + Optional[int], |
| 212 | + Option( |
| 213 | + help="Maximum number of samples to evaluate per task. Useful for quick testing or debugging. If None, evaluates on all available samples.", |
| 214 | + rich_help_panel=HELP_PANEL_NAME_3, |
| 215 | + ), |
| 216 | + ], |
| 217 | + default=None, |
| 218 | +) |
| 219 | + |
| 220 | +job_id = Arg( |
| 221 | + type=Annotated[ |
| 222 | + int, |
| 223 | + Option( |
| 224 | + help="Optional job identifier for tracking and organizing multiple evaluation runs. Useful in cluster environments.", |
| 225 | + rich_help_panel=HELP_PANEL_NAME_3, |
| 226 | + ), |
| 227 | + ], |
| 228 | + default=0, |
| 229 | +) |
| 230 | + |
| 231 | + |
| 232 | +# Common argument patterns |
| 233 | +tasks = Arg( |
| 234 | + type=Annotated[ |
| 235 | + str, |
| 236 | + Argument( |
| 237 | + help="Comma-separated list of tasks to evaluate. Format: 'task1,task2' or 'suite|task|version|split'. Use 'lighteval tasks list' to see available tasks." |
| 238 | + ), |
| 239 | + ], |
| 240 | + default=None, # Required argument, no default |
| 241 | +) |
| 242 | + |
| 243 | +model_args = Arg( |
| 244 | + type=Annotated[ |
| 245 | + str, |
| 246 | + Argument( |
| 247 | + help="Model configuration in key=value format (e.g., 'pretrained=model_name,device=cuda') or path to YAML config file. See examples/model_configs/ for template files." |
| 248 | + ), |
| 249 | + ], |
| 250 | + default=None, # Required argument, no default |
| 251 | +) |
0 commit comments