feat(BA-7317): render prometheus query preset templates with Jinja

changes/13675.breaking.md

@@ -0,0 +1 @@
+Switch Prometheus query preset templates to sandboxed Jinja syntax ({{ labels }}, {{ window }}, {{ group_by }}) with automatic migration of stored presets; the legacy str.format placeholder syntax is no longer accepted

docs/manager/graphql-reference/supergraph.graphql

@@ -4556,8 +4556,10 @@
   """Prometheus metric name."""
   metricName: String!
 
-  """PromQL template with {labels}, {window}, {group_by} placeholders."""
+  """
+  PromQL template with Jinja placeholders ({{ labels }}, {{ window }}, {{ group_by }}).
+  """
   queryTemplate: String!
 
   """Default time window."""
   timeWindow: String = null

docs/manager/graphql-reference/v2-schema.graphql

@@ -3068,7 +3068,9 @@ input CreateQueryDefinitionInput {
   """Prometheus metric name."""
   metricName: String!
 
-  """PromQL template with {labels}, {window}, {group_by} placeholders."""
+  """
+  PromQL template with Jinja placeholders ({{ labels }}, {{ window }}, {{ group_by }}).
+  """
   queryTemplate: String!
 
   """Default time window."""

docs/manager/rest-reference/openapi.json

@@ -17143,7 +17143,7 @@
             "type": "string"
           },
           "query_template": {
-            "description": "PromQL template with placeholders",
+            "description": "PromQL template with Jinja placeholders ({{ labels }}, {{ window }}, {{ group_by }})",
             "title": "Query Template",
             "type": "string"
           },
@@ -17481,7 +17481,7 @@
               }
             ],
             "default": null,
-            "description": "Updated PromQL template with placeholders",
+            "description": "Updated PromQL template with Jinja placeholders ({{ labels }}, {{ window }}, {{ group_by }})",
             "title": "Query Template"
           },
           "time_window": {

fixtures/manager/example-prometheus-query-presets.json

@@ -7,7 +7,7 @@
       "rank": 100,
       "category_name": "container",
       "metric_name": "backendai_container_utilization",
-      "query_template": "sum by ({group_by})(backendai_container_utilization{{{labels}}})",
+      "query_template": "sum by ({{ group_by }})(backendai_container_utilization{ {{ labels }}})",
       "time_window": null,
       "options": {
         "filter_labels": [
@@ -36,7 +36,7 @@
       "rank": 110,
       "category_name": "container",
       "metric_name": "backendai_container_utilization",
-      "query_template": "avg by ({group_by})(backendai_container_utilization{{{labels}}})",
+      "query_template": "avg by ({{ group_by }})(backendai_container_utilization{ {{ labels }}})",
       "time_window": null,
       "options": {
         "filter_labels": [
@@ -65,7 +65,7 @@
       "rank": 120,
       "category_name": "container",
       "metric_name": "backendai_container_utilization",
-      "query_template": "max by ({group_by})(backendai_container_utilization{{{labels}}})",
+      "query_template": "max by ({{ group_by }})(backendai_container_utilization{ {{ labels }}})",
       "time_window": null,
       "options": {
         "filter_labels": [
@@ -94,7 +94,7 @@
       "rank": 130,
       "category_name": "container",
       "metric_name": "backendai_container_utilization",
-      "query_template": "min by ({group_by})(backendai_container_utilization{{{labels}}})",
+      "query_template": "min by ({{ group_by }})(backendai_container_utilization{ {{ labels }}})",
       "time_window": null,
       "options": {
         "filter_labels": [
@@ -123,7 +123,7 @@
       "rank": 200,
       "category_name": "container",
       "metric_name": "backendai_container_utilization",
-      "query_template": "sum by ({group_by})(rate(backendai_container_utilization{{{labels}}}[{window}]))",
+      "query_template": "sum by ({{ group_by }})(rate(backendai_container_utilization{ {{ labels }}}[{{ window }}]))",
       "time_window": "5m",
       "options": {
         "filter_labels": [
@@ -152,7 +152,7 @@
       "rank": 210,
       "category_name": "container",
       "metric_name": "backendai_container_utilization",
-      "query_template": "avg by ({group_by})(rate(backendai_container_utilization{{{labels}}}[{window}]))",
+      "query_template": "avg by ({{ group_by }})(rate(backendai_container_utilization{ {{ labels }}}[{{ window }}]))",
       "time_window": "5m",
       "options": {
         "filter_labels": [
@@ -181,7 +181,7 @@
       "rank": 220,
       "category_name": "container",
       "metric_name": "backendai_container_utilization",
-      "query_template": "max by ({group_by})(rate(backendai_container_utilization{{{labels}}}[{window}]))",
+      "query_template": "max by ({{ group_by }})(rate(backendai_container_utilization{ {{ labels }}}[{{ window }}]))",
       "time_window": "5m",
       "options": {
         "filter_labels": [
@@ -210,7 +210,7 @@
       "rank": 230,
       "category_name": "container",
       "metric_name": "backendai_container_utilization",
-      "query_template": "min by ({group_by})(rate(backendai_container_utilization{{{labels}}}[{window}]))",
+      "query_template": "min by ({{ group_by }})(rate(backendai_container_utilization{ {{ labels }}}[{{ window }}]))",
       "time_window": "5m",
       "options": {
         "filter_labels": [
@@ -239,7 +239,7 @@
       "rank": 300,
       "category_name": "vllm-inference",
       "metric_name": "vllm:num_requests_running",
-      "query_template": "sum by ({group_by})(vllm:num_requests_running{{{labels}}})",
+      "query_template": "sum by ({{ group_by }})(vllm:num_requests_running{ {{ labels }}})",
       "time_window": null,
       "options": {
         "filter_labels": [
@@ -257,7 +257,7 @@
       "rank": 310,
       "category_name": "vllm-inference",
       "metric_name": "vllm:num_requests_running",
-      "query_template": "avg by ({group_by})(vllm:num_requests_running{{{labels}}})",
+      "query_template": "avg by ({{ group_by }})(vllm:num_requests_running{ {{ labels }}})",
       "time_window": null,
       "options": {
         "filter_labels": [
@@ -275,7 +275,7 @@
       "rank": 320,
       "category_name": "vllm-inference",
       "metric_name": "vllm:num_requests_running",
-      "query_template": "max by ({group_by})(vllm:num_requests_running{{{labels}}})",
+      "query_template": "max by ({{ group_by }})(vllm:num_requests_running{ {{ labels }}})",
       "time_window": null,
       "options": {
         "filter_labels": [
@@ -293,7 +293,7 @@
       "rank": 400,
       "category_name": "vllm-inference",
       "metric_name": "vllm:num_requests_waiting",
-      "query_template": "sum by ({group_by})(vllm:num_requests_waiting{{{labels}}})",
+      "query_template": "sum by ({{ group_by }})(vllm:num_requests_waiting{ {{ labels }}})",
       "time_window": null,
       "options": {
         "filter_labels": [
@@ -311,7 +311,7 @@
       "rank": 410,
       "category_name": "vllm-inference",
       "metric_name": "vllm:num_requests_waiting",
-      "query_template": "avg by ({group_by})(vllm:num_requests_waiting{{{labels}}})",
+      "query_template": "avg by ({{ group_by }})(vllm:num_requests_waiting{ {{ labels }}})",
       "time_window": null,
       "options": {
         "filter_labels": [
@@ -329,7 +329,7 @@
       "rank": 420,
       "category_name": "vllm-inference",
       "metric_name": "vllm:num_requests_waiting",
-      "query_template": "max by ({group_by})(vllm:num_requests_waiting{{{labels}}})",
+      "query_template": "max by ({{ group_by }})(vllm:num_requests_waiting{ {{ labels }}})",
       "time_window": null,
       "options": {
         "filter_labels": [
@@ -347,7 +347,7 @@
       "rank": 500,
       "category_name": "vllm-inference",
       "metric_name": "vllm:gpu_cache_usage_perc",
-      "query_template": "avg by ({group_by})(vllm:gpu_cache_usage_perc{{{labels}}})",
+      "query_template": "avg by ({{ group_by }})(vllm:gpu_cache_usage_perc{ {{ labels }}})",
       "time_window": null,
       "options": {
         "filter_labels": [
@@ -365,7 +365,7 @@
       "rank": 510,
       "category_name": "vllm-inference",
       "metric_name": "vllm:gpu_cache_usage_perc",
-      "query_template": "max by ({group_by})(vllm:gpu_cache_usage_perc{{{labels}}})",
+      "query_template": "max by ({{ group_by }})(vllm:gpu_cache_usage_perc{ {{ labels }}})",
       "time_window": null,
       "options": {
         "filter_labels": [
@@ -383,7 +383,7 @@
       "rank": 520,
       "category_name": "vllm-inference",
       "metric_name": "vllm:gpu_cache_usage_perc",
-      "query_template": "min by ({group_by})(vllm:gpu_cache_usage_perc{{{labels}}})",
+      "query_template": "min by ({{ group_by }})(vllm:gpu_cache_usage_perc{ {{ labels }}})",
       "time_window": null,
       "options": {
         "filter_labels": [
@@ -401,7 +401,7 @@
       "rank": 600,
       "category_name": "vllm-inference",
       "metric_name": "vllm:request_success_total",
-      "query_template": "sum by ({group_by})(rate(vllm:request_success_total{{{labels}}}[{window}]))",
+      "query_template": "sum by ({{ group_by }})(rate(vllm:request_success_total{ {{ labels }}}[{{ window }}]))",
       "time_window": "5m",
       "options": {
         "filter_labels": [
@@ -419,7 +419,7 @@
       "rank": 610,
       "category_name": "vllm-inference",
       "metric_name": "vllm:request_success_total",
-      "query_template": "avg by ({group_by})(rate(vllm:request_success_total{{{labels}}}[{window}]))",
+      "query_template": "avg by ({{ group_by }})(rate(vllm:request_success_total{ {{ labels }}}[{{ window }}]))",
       "time_window": "5m",
       "options": {
         "filter_labels": [
@@ -437,7 +437,7 @@
       "rank": 620,
       "category_name": "vllm-inference",
       "metric_name": "vllm:request_success_total",
-      "query_template": "max by ({group_by})(rate(vllm:request_success_total{{{labels}}}[{window}]))",
+      "query_template": "max by ({{ group_by }})(rate(vllm:request_success_total{ {{ labels }}}[{{ window }}]))",
       "time_window": "5m",
       "options": {
         "filter_labels": [
@@ -455,7 +455,7 @@
       "rank": 700,
       "category_name": "vllm-inference",
       "metric_name": "vllm:e2e_request_latency_seconds",
-      "query_template": "avg by ({group_by})(rate(vllm:e2e_request_latency_seconds_sum{{{labels}}}[{window}]) / rate(vllm:e2e_request_latency_seconds_count{{{labels}}}[{window}]))",
+      "query_template": "avg by ({{ group_by }})(rate(vllm:e2e_request_latency_seconds_sum{ {{ labels }}}[{{ window }}]) / rate(vllm:e2e_request_latency_seconds_count{ {{ labels }}}[{{ window }}]))",
       "time_window": "5m",
       "options": {
         "filter_labels": [
@@ -473,7 +473,7 @@
       "rank": 710,
       "category_name": "vllm-inference",
       "metric_name": "vllm:e2e_request_latency_seconds",
-      "query_template": "max by ({group_by})(rate(vllm:e2e_request_latency_seconds_sum{{{labels}}}[{window}]) / rate(vllm:e2e_request_latency_seconds_count{{{labels}}}[{window}]))",
+      "query_template": "max by ({{ group_by }})(rate(vllm:e2e_request_latency_seconds_sum{ {{ labels }}}[{{ window }}]) / rate(vllm:e2e_request_latency_seconds_count{ {{ labels }}}[{{ window }}]))",
       "time_window": "5m",
       "options": {
         "filter_labels": [

src/ai/backend/common/data/idle_checker/types.py

@@ -48,12 +48,12 @@ class UtilizationThresholdEntry(BackendAISchema):
     )
     filter_labels: list[MetricLabel] = Field(
         default_factory=list,
-        description="Label filters injected into the preset's {labels} placeholder.",
+        description="Label filters injected into the preset's {{ labels }} placeholder.",
     )
     group_labels: list[str] = Field(
         default_factory=lambda: [SESSION_ID_LABEL],
         description=(
-            "Labels injected into the preset's {group_by} placeholder. "
+            "Labels injected into the preset's {{ group_by }} placeholder. "
             "Must include 'session_id' for per-session values to be mapped. "
             "When 'session_id' is grouped, the checker adds a session_id filter that "
             "limits the query to the sessions being evaluated; a user-provided "

src/ai/backend/common/dto/manager/v2/prometheus_query_preset/request.py

@@ -65,7 +65,11 @@ class CreateQueryDefinitionInput(BaseRequestModel):
     rank: int = Field(default=0, ge=0, description="Sort rank (lower = higher priority)")
     category_id: UUID | None = Field(default=None, description="Category ID")
     metric_name: str = Field(description="Prometheus metric name")
-    query_template: str = Field(description="PromQL template with placeholders")
+    query_template: str = Field(
+        description=(
+            "PromQL template with Jinja placeholders ({{ labels }}, {{ window }}, {{ group_by }})"
+        )
+    )
     time_window: str | None = Field(
         default=None,
         pattern=PROMETHEUS_DURATION_PATTERN,
@@ -122,7 +126,11 @@ class ModifyQueryDefinitionInput(BaseRequestModel):
     )
     metric_name: str | None = Field(default=None, description="Updated Prometheus metric name")
     query_template: str | None = Field(
-        default=None, description="Updated PromQL template with placeholders"
+        default=None,
+        description=(
+            "Updated PromQL template with Jinja placeholders"
+            " ({{ labels }}, {{ window }}, {{ group_by }})"
+        ),
     )
     time_window: str | Sentinel | None = Field(
         default=SENTINEL,

src/ai/backend/common/dto/manager/v2/prometheus_query_preset/validators.py

@@ -1,66 +1,70 @@
-"""Request DTO validators for prometheus_query_preset templates."""
+"""Validators for prometheus_query_preset templates."""
 
 from __future__ import annotations
 
 import re
+from collections.abc import Iterator
+
+from jinja2 import StrictUndefined, TemplateError, TemplateSyntaxError, nodes
+from jinja2.sandbox import ImmutableSandboxedEnvironment
 
 from ai.backend.common.exception import InvalidMetricPresetTemplate
 
 __all__ = (
     "PLACEHOLDER_NAMES",
-    "escape_non_placeholders",
+    "PROMQL_TEMPLATE_ENV",
     "validate_query_template",
 )
 
 PLACEHOLDER_NAMES = frozenset({"labels", "window", "group_by"})
 
-_BRACE_BLOCK_RE = re.compile(r"\{([^{}]*)\}")
-_UNSUPPORTED_TEMPLATE_VAR_RE = re.compile(r"\$\{[^}]+\}|\$[A-Za-z_][A-Za-z0-9_]*")
+# Sandboxed: templates are user input from the admin API.
+PROMQL_TEMPLATE_ENV = ImmutableSandboxedEnvironment(undefined=StrictUndefined)
 
+# Literal text and `{{ placeholder }}` substitution only.
+_ALLOWED_NODE_TYPES = (nodes.Template, nodes.Output, nodes.TemplateData, nodes.Name)
 
-def escape_non_placeholders(template: str) -> str:
-    """Normalize each ``{X}`` so ``str.format`` produces a single PromQL ``{value}``
-    regardless of how many braces the user wrote.
-    """
+_UNSUPPORTED_TEMPLATE_VAR_RE = re.compile(r"\$\{[^}]+\}|\$[A-Za-z_][A-Za-z0-9_]*")
+# Bare `{placeholder}` or any `{{{`: the pre-Jinja str.format syntax.
+_LEGACY_TEMPLATE_RE = re.compile(r"(?<!\{)\{(?:labels|window|group_by)\}(?!\})|\{\{\{")
 
-    def repl(match: re.Match[str]) -> str:
-        name = match.group(1)
-        start, end = match.span()
-        text = match.string
-        already_wrapped = (
-            start > 0 and text[start - 1] == "{" and end < len(text) and text[end] == "}"
-        )
-        inside_escaped_braces = (
-            text.rfind("{{", 0, start) > text.rfind("}}", 0, start) and text.find("}}", end) != -1
-        )
-        if name not in PLACEHOLDER_NAMES:
-            return match.group(0) if already_wrapped else "{{" + name + "}}"
-        if name != "labels":
-            return match.group(0)
-        return (
-            match.group(0)
-            if already_wrapped or inside_escaped_braces
-            else "{{" + match.group(0) + "}}"
-        )
 
-    return _BRACE_BLOCK_RE.sub(repl, template)
+def _walk(node: nodes.Node) -> Iterator[nodes.Node]:
+    yield node
+    for child in node.iter_child_nodes():
+        yield from _walk(child)
 
 
-def validate_query_template(template: str) -> str:
-    """Reject empty templates, foreign variables, or malformed braces."""
+def validate_query_template(template: str) -> None:
+    """Validate a Jinja PromQL template; raises ``InvalidMetricPresetTemplate``."""
     if not template.strip():
         raise InvalidMetricPresetTemplate("Template must not be empty.")
     unsupported_vars = _UNSUPPORTED_TEMPLATE_VAR_RE.findall(template)
     if unsupported_vars:
-        placeholders = ", ".join(f"{{{name}}}" for name in sorted(PLACEHOLDER_NAMES))
+        placeholders = ", ".join(f"{{{{ {name} }}}}" for name in sorted(PLACEHOLDER_NAMES))
         raise InvalidMetricPresetTemplate(
             f"Unsupported template variables: {unsupported_vars}. "
             f"Use placeholders {placeholders} or literal PromQL values."
         )
+    if _LEGACY_TEMPLATE_RE.search(template):
+        raise InvalidMetricPresetTemplate(
+            "Legacy str.format template syntax is no longer supported; "
+            f"use {{{{ labels }}}}, {{{{ window }}}}, {{{{ group_by }}}}: {template!r}"
+        )
+    try:
+        ast = PROMQL_TEMPLATE_ENV.parse(template)
+    except TemplateSyntaxError as e:
+        raise InvalidMetricPresetTemplate(f"Invalid template syntax ({e}): {template!r}") from e
+    for node in _walk(ast):
+        if not isinstance(node, _ALLOWED_NODE_TYPES):
+            raise InvalidMetricPresetTemplate(
+                f"Only {{{{ placeholder }}}} substitution is allowed; "
+                f"found {type(node).__name__}: {template!r}"
+            )
     try:
-        escape_non_placeholders(template).format(labels="", window="", group_by="")
-    except (ValueError, KeyError, IndexError) as e:
+        # Smoke-render with empty values; StrictUndefined rejects unknown variables.
+        PROMQL_TEMPLATE_ENV.from_string(template).render(labels="", window="", group_by="")
+    except TemplateError as e:
         raise InvalidMetricPresetTemplate(
             f"Failed to render PromQL template ({type(e).__name__}: {e}): {template!r}"
         ) from e
-    return template

src/ai/backend/manager/api/gql/prometheus_query_preset/types/inputs.py

@@ -61,7 +61,9 @@ class CreateQueryDefinitionInput(PydanticInputMixin[CreateQueryDefinitionInputDT
     category_id: UUID | None = gql_field(description="Category UUID.", default=None)
     metric_name: str = gql_field(description="Prometheus metric name.")
     query_template: str = gql_field(
-        description="PromQL template with {labels}, {window}, {group_by} placeholders."
+        description=(
+            "PromQL template with Jinja placeholders ({{ labels }}, {{ window }}, {{ group_by }})."
+        )
     )
     time_window: str | None = gql_field(description="Default time window.", default=None)
     options: QueryDefinitionOptionsInput = gql_field(