Add comprehensive embeddings benchmark support

Implements full embeddings benchmarking capability
including schemas, quality validation (cosine similarity,
MTEB), output formatters (CSV, HTML, JSON, console),
mock server handler, CLI integration, and comprehensive
test suite.

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
Signed-off-by: Maryam Tahhan <mtahhan@redhat.com>

Author: maryamtahhan

pyproject.toml

@@ -74,7 +74,7 @@ dependencies = [
 
 [project.optional-dependencies]
 # Meta Extras
-all = ["guidellm[perf,tokenizers,audio,vision]"]
+all = ["guidellm[perf,tokenizers,audio,vision,embeddings]"]
 recommended = ["guidellm[perf,tokenizers]"]
 # Feature Extras
 perf = ["orjson", "msgpack", "msgspec", "uvloop"]
@@ -90,6 +90,12 @@ vision = [
     "datasets[vision]",
     "pillow",
 ]
+embeddings = [
+    # Quality validation with baseline models
+    "sentence-transformers>=2.2.0",
+    # MTEB benchmark integration
+    "mteb>=1.0.0",
+]
 # Dev Tooling
 dev = [
     # Install all optional dependencies
@@ -179,7 +185,9 @@ module = [
     "transformers.*",
     "setuptools.*",
     "setuptools_git_versioning.*",
-    "torchcodec.*"
+    "torchcodec.*",
+    "sentence_transformers.*",
+    "mteb.*"
 ]
 ignore_missing_imports = true
 

src/guidellm/__main__.py

@@ -794,3 +794,226 @@ def mock_server(
 
 if __name__ == "__main__":
     cli()
+
+
+@benchmark.command(
+    "embeddings",
+    help=(
+        "Run embeddings benchmark with optional quality validation. "
+        "Supports cosine similarity validation and MTEB benchmark evaluation."
+    ),
+    context_settings={"auto_envvar_prefix": "GUIDELLM"},
+)
+@click.option(
+    "--target",
+    type=str,
+    required=True,
+    help="Target backend URL (e.g., http://localhost:8000).",
+)
+@click.option(
+    "--data",
+    type=str,
+    multiple=True,
+    required=True,
+    help=(
+        "HuggingFace dataset ID, path to dataset, path to data file "
+        "(csv/json/jsonl/txt), or synthetic data config."
+    ),
+)
+@click.option(
+    "--profile",
+    default="sweep",
+    type=click.Choice(STRATEGY_PROFILE_CHOICES),
+    help=f"Benchmark profile type. Options: {', '.join(STRATEGY_PROFILE_CHOICES)}.",
+)
+@click.option(
+    "--rate",
+    callback=cli_tools.parse_list_floats,
+    multiple=True,
+    default=None,
+    help="Benchmark rate(s) to test. Meaning depends on profile.",
+)
+@click.option(
+    "--backend",
+    type=click.Choice(list(get_literal_vals(BackendType))),
+    default="openai_http",
+    help=f"Backend type. Options: {', '.join(get_literal_vals(BackendType))}.",
+)
+@click.option(
+    "--backend-kwargs",
+    callback=cli_tools.parse_json,
+    default=None,
+    help='JSON string of backend arguments. E.g., \'{"api_key": "key"}\'',
+)
+@click.option(
+    "--model",
+    default=None,
+    type=str,
+    help="Model ID to benchmark. If not provided, uses first available model.",
+)
+@click.option(
+    "--request-format",
+    default="embeddings",
+    help="Format to use for requests (default: embeddings).",
+)
+@click.option(
+    "--processor",
+    default=None,
+    type=str,
+    help="Processor or tokenizer for token counts. If not provided, loads from model.",
+)
+@click.option(
+    "--data-samples",
+    default=-1,
+    type=int,
+    help="Number of samples from dataset. -1 (default) uses all samples.",
+)
+@click.option(
+    "--outputs",
+    default=["json", "csv", "html"],
+    callback=cli_tools.parse_csv,
+    help=(
+        "Comma-separated list of output formats: json,csv,html,console. "
+        "Default: json,csv,html"
+    ),
+)
+@click.option(
+    "--output-dir",
+    type=click.Path(file_okay=False, dir_okay=True, path_type=Path),
+    default=Path.cwd(),
+    help="Directory to save output files. Default: current directory.",
+)
+@click.option(
+    "--max-requests",
+    default=None,
+    type=int,
+    help="Maximum number of requests to execute.",
+)
+@click.option(
+    "--max-errors",
+    default=None,
+    type=int,
+    help="Maximum number of errors before stopping benchmark.",
+)
+@click.option(
+    "--max-duration",
+    default=None,
+    type=float,
+    help="Maximum duration in seconds for benchmark execution.",
+)
+# Embeddings-specific quality validation options
+@click.option(
+    "--enable-quality-validation",
+    is_flag=True,
+    default=False,
+    help="Enable quality validation using cosine similarity against baseline model.",
+)
+@click.option(
+    "--baseline-model",
+    default=None,
+    type=str,
+    help=(
+        "HuggingFace model for baseline comparison. "
+        "E.g., 'sentence-transformers/all-MiniLM-L6-v2'. "
+        "Defaults to target model if not specified."
+    ),
+)
+@click.option(
+    "--quality-tolerance",
+    default=1e-2,
+    type=float,
+    help=(
+        "Cosine similarity tolerance threshold. "
+        "Default: 1e-2 (standard), use 5e-4 for MTEB-level validation."
+    ),
+)
+@click.option(
+    "--enable-mteb",
+    is_flag=True,
+    default=False,
+    help="Enable MTEB benchmark evaluation for standardized quality scoring.",
+)
+@click.option(
+    "--mteb-tasks",
+    callback=cli_tools.parse_csv,
+    default=None,
+    help=(
+        "Comma-separated list of MTEB tasks. "
+        "Default: STS12,STS13,STSBenchmark. E.g., 'STS12,STS13,STS14'"
+    ),
+)
+@click.option(
+    "--encoding-format",
+    type=click.Choice(["float", "base64"]),
+    default="float",
+    help="Embedding encoding format. Options: float, base64. Default: float.",
+)
+@click.option(
+    "--disable-console",
+    is_flag=True,
+    default=False,
+    help="Disable all console output (including progress display).",
+)
+@click.option(
+    "--disable-console-interactive",
+    is_flag=True,
+    default=False,
+    help="Disable interactive console elements (progress bar, tables).",
+)
+@click.option(
+    "--random-seed",
+    default=42,
+    type=int,
+    help="Random seed for reproducibility. Default: 42.",
+)
+def embeddings(**kwargs):
+    """Run embeddings benchmark with optional quality validation."""
+    from guidellm.benchmark.embeddings_entrypoints import benchmark_embeddings
+    from guidellm.benchmark.schemas.embeddings import BenchmarkEmbeddingsArgs
+
+    # Only set CLI args that differ from click defaults
+    kwargs = cli_tools.set_if_not_default(click.get_current_context(), **kwargs)
+
+    # Handle console options
+    disable_console = kwargs.pop("disable_console", False)
+    disable_console_interactive = (
+        kwargs.pop("disable_console_interactive", False) or disable_console
+    )
+    console = Console() if not disable_console else None
+
+    envs = cli_tools.list_set_env()
+    if console and envs:
+        console.print_update(
+            title=(
+                "Note: the following environment variables "
+                "are set and **may** affect configuration"
+            ),
+            details=", ".join(envs),
+            status="warning",
+        )
+
+    try:
+        args = BenchmarkEmbeddingsArgs.create(scenario=None, **kwargs)
+    except ValidationError as err:
+        errs = err.errors(include_url=False, include_context=True, include_input=True)
+        param_name = "--" + str(errs[0]["loc"][0]).replace("_", "-")
+        raise click.BadParameter(
+            errs[0]["msg"], ctx=click.get_current_context(), param_hint=param_name
+        ) from err
+
+    if uvloop is not None:
+        asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
+
+    asyncio.run(
+        benchmark_embeddings(
+            args=args,
+            progress=(
+                GenerativeConsoleBenchmarkerProgress()
+                if not disable_console_interactive
+                else None
+            ),
+            console=console,
+        )
+    )
+
+

src/guidellm/backends/openai/request_handlers.py

@@ -20,6 +20,7 @@
 __all__ = [
     "AudioRequestHandler",
     "ChatCompletionsRequestHandler",
+    "EmbeddingsRequestHandler",
     "OpenAIRequestHandler",
     "OpenAIRequestHandlerFactory",
     "TextCompletionsRequestHandler",
@@ -667,3 +668,113 @@ def extract_metrics(
             text_words=len(text.split()) if text else 0,
             text_characters=len(text) if text else 0,
         )
+
+
+@OpenAIRequestHandlerFactory.register("embeddings")
+class EmbeddingsRequestHandler(OpenAIRequestHandler):
+    """
+    Request handler for OpenAI-style embeddings endpoints.
+
+    Handles embeddings requests which do not support streaming and return
+    embedding vectors instead of generated text. Processes input text into
+    embeddings with optional quality validation support.
+    """
+
+    def format(
+        self,
+        data: GenerationRequest,
+        **kwargs,
+    ) -> GenerationRequestArguments:
+        """
+        Format the embeddings generation request.
+
+        :param data: The generation request to format
+        :param **kwargs: Additional keyword arguments (model, encoding_format, etc.)
+        :return: The formatted request arguments
+        """
+        arguments = GenerationRequestArguments()
+        arguments.body = {}
+        arguments.stream = False  # Embeddings never stream
+
+        # Add model
+        if kwargs.get("model") is not None:
+            arguments.body["model"] = kwargs["model"]
+
+        # Build input from text columns
+        input_texts = []
+        for text in data.columns.get("text_column", []):
+            if text:
+                input_texts.append(text)
+
+        # Use single string if only one text, otherwise list
+        if len(input_texts) == 1:
+            arguments.body["input"] = input_texts[0]
+        else:
+            arguments.body["input"] = input_texts
+
+        # Add optional parameters
+        if kwargs.get("encoding_format"):
+            arguments.body["encoding_format"] = kwargs["encoding_format"]
+        if kwargs.get("dimensions"):
+            arguments.body["dimensions"] = kwargs["dimensions"]
+        if kwargs.get("truncate_prompt_tokens"):
+            arguments.body["truncate_prompt_tokens"] = kwargs["truncate_prompt_tokens"]
+
+        # Apply extra arguments
+        if kwargs.get("extras"):
+            arguments.body.update(kwargs["extras"])
+
+        return arguments
+
+    def compile_non_streaming(
+        self,
+        request: GenerationRequest,
+        arguments: GenerationRequestArguments,
+        response: Any,
+    ) -> GenerationResponse:
+        """
+        Process a complete non-streaming embeddings API response.
+
+        :param request: Original generation request
+        :param arguments: Request arguments used
+        :param response: Raw API response data
+        :return: GenerationResponse with embeddings data
+        """
+        # Extract embeddings data
+        embeddings_data = response.get("data", [])
+        usage = response.get("usage", {})
+
+        # Build response (no text output for embeddings)
+        return GenerationResponse(
+            request_id=request.request_id,
+            text="",  # Embeddings don't generate text
+            input_metrics=UsageMetrics(
+                text_tokens=usage.get("prompt_tokens", 0),
+            ),
+            output_metrics=UsageMetrics(
+                text_tokens=0,  # No output tokens for embeddings
+            ),
+        )
+
+    def add_streaming_line(self, line: str) -> int | None:
+        """
+        Embeddings do not support streaming.
+
+        :param line: Streaming line (unused)
+        :return: None (not supported)
+        :raises NotImplementedError: Embeddings never stream
+        """
+        raise NotImplementedError("Embeddings do not support streaming")
+
+    def compile_streaming(
+        self, request: GenerationRequest, arguments: GenerationRequestArguments
+    ) -> GenerationResponse:
+        """
+        Embeddings do not support streaming.
+
+        :param request: Generation request (unused)
+        :param arguments: Request arguments (unused)
+        :return: Never returns
+        :raises NotImplementedError: Embeddings never stream
+        """
+        raise NotImplementedError("Embeddings do not support streaming")