fix typing, remove dataset preprocessing until it's migrated

Author: markurtz

src/guidellm/__main__.py

@@ -45,26 +45,12 @@
     reimport_benchmarks_report,
 )
 from guidellm.mock_server import MockServer, MockServerConfig
-from guidellm.preprocess.dataset import ShortPromptStrategy, process_dataset
 from guidellm.scheduler import StrategyType
 from guidellm.schemas import GenerativeRequestType
 from guidellm.settings import print_config
 from guidellm.utils import Console, DefaultGroupHandler, get_literal_vals
 from guidellm.utils import cli as cli_tools
 
-__all__ = [
-    "STRATEGY_PROFILE_CHOICES",
-    "benchmark",
-    "cli",
-    "config",
-    "dataset",
-    "decode_escaped_str",
-    "from_file",
-    "mock_server",
-    "preprocess",
-    "run",
-]
-
 STRATEGY_PROFILE_CHOICES: list[str] = list(get_literal_vals(ProfileType | StrategyType))
 """Available strategy and profile type choices for benchmark execution."""
 
@@ -469,128 +455,6 @@ def preprocess():
     """Dataset preprocessing utilities."""
 
 
-@preprocess.command(
-    "dataset",
-    help=(
-        "Process a dataset to have specific prompt and output token sizes. "
-        "Supports multiple strategies for handling prompts and optional "
-        "Hugging Face Hub upload.\n\n"
-        "DATA: Path to the input dataset or dataset ID.\n\n"
-        "OUTPUT_PATH: Path to save the processed dataset, including file suffix."
-    ),
-    context_settings={"auto_envvar_prefix": "GUIDELLM"},
-)
-@click.argument(
-    "data",
-    type=str,
-    required=True,
-)
-@click.argument(
-    "output_path",
-    type=click.Path(file_okay=True, dir_okay=False, writable=True, resolve_path=True),
-    required=True,
-)
-@click.option(
-    "--processor",
-    type=str,
-    required=True,
-    help="Processor or tokenizer name for calculating token counts.",
-)
-@click.option(
-    "--processor-args",
-    default=None,
-    callback=cli_tools.parse_json,
-    help="JSON string of arguments to pass to the processor constructor.",
-)
-@click.option(
-    "--data-args",
-    callback=cli_tools.parse_json,
-    help="JSON string of arguments to pass to dataset creation.",
-)
-@click.option(
-    "--short-prompt-strategy",
-    type=click.Choice([s.value for s in ShortPromptStrategy]),
-    default=ShortPromptStrategy.IGNORE.value,
-    show_default=True,
-    help="Strategy for handling prompts shorter than target length.",
-)
-@click.option(
-    "--pad-char",
-    type=str,
-    default="",
-    callback=decode_escaped_str,
-    help="Character to pad short prompts with when using 'pad' strategy.",
-)
-@click.option(
-    "--concat-delimiter",
-    type=str,
-    default="",
-    help=(
-        "Delimiter for concatenating short prompts (used with 'concatenate' strategy)."
-    ),
-)
-@click.option(
-    "--prompt-tokens",
-    type=str,
-    default=None,
-    help="Prompt tokens configuration (JSON, YAML file, or key=value string).",
-)
-@click.option(
-    "--output-tokens",
-    type=str,
-    default=None,
-    help="Output tokens configuration (JSON, YAML file, or key=value string).",
-)
-@click.option(
-    "--push-to-hub",
-    is_flag=True,
-    help="Push the processed dataset to Hugging Face Hub.",
-)
-@click.option(
-    "--hub-dataset-id",
-    type=str,
-    default=None,
-    help=("Hugging Face Hub dataset ID for upload (required if --push-to-hub is set)."),
-)
-@click.option(
-    "--random-seed",
-    type=int,
-    default=42,
-    show_default=True,
-    help="Random seed for reproducible token sampling.",
-)
-def dataset(
-    data,
-    output_path,
-    processor,
-    processor_args,
-    data_args,
-    short_prompt_strategy,
-    pad_char,
-    concat_delimiter,
-    prompt_tokens,
-    output_tokens,
-    push_to_hub,
-    hub_dataset_id,
-    random_seed,
-):
-    process_dataset(
-        data=data,
-        output_path=output_path,
-        processor=processor,
-        prompt_tokens=prompt_tokens,
-        output_tokens=output_tokens,
-        processor_args=processor_args,
-        data_args=data_args,
-        short_prompt_strategy=short_prompt_strategy,
-        pad_char=pad_char,
-        concat_delimiter=concat_delimiter,
-        push_to_hub=push_to_hub,
-        hub_dataset_id=hub_dataset_id,
-        random_seed=random_seed,
-    )
-
-
 @cli.command(
     "mock-server",
     help=(

src/guidellm/preprocess/dataset.py

@@ -276,8 +276,8 @@ def process_dataset(
         processor_args,
         "dataset conversion.",
     )
-    prompt_column = column_mappings.get("prompt_column")
-    output_column = column_mappings.get(
+    prompt_column = column_mappings.get("prompt_column")  # type: ignore[attr-defined]
+    output_column = column_mappings.get(  # type: ignore[attr-defined]
         "output_tokens_count_column", "output_tokens_count"
     )
 
@@ -304,7 +304,7 @@ def process_dataset(
         )
     )
 
-    dataset_iterator = iter(dataset)
+    dataset_iterator = iter(dataset)  # type: ignore[call-overload]
     processed_prompts = []
     prompt_handler = STRATEGY_HANDLERS[short_prompt_strategy]