Merge branch 'main' into llong/dit_ag_min

Author: llongTT

.github/CODEOWNERS

@@ -244,6 +244,8 @@ tests/ttnn/**/operations/pool/ @tenstorrent/metalium-developers-convolutions @te
 tests/sweep_framework/ @stevendae @bbradelTT @ntarafdar @pavlejosipovic @tenstorrent/codeowner-bypass
 tests/ttnn/nightly/unit_tests/operations/matmul/ @tenstorrent/metalium-developers-mmfusedreduce @tenstorrent/codeowner-bypass
 tests/ttnn/nightly/unit_tests/operations/transformers/ @tenstorrent/metallium-maintainers-llama-models @tenstorrent/codeowner-bypass
+tests/ttnn/unit_tests/operations/sdpa/ @tenstorrent/metallium-maintainers-llama-models @tenstorrent/codeowner-bypass
+tests/ttnn/nightly/unit_tests/operations/sdpa/ @tenstorrent/metallium-maintainers-llama-models @tenstorrent/codeowner-bypass
 tests/ttnn/nightly/unit_tests/operations/eltwise/ @tenstorrent/metalium-developers-eltwise @tenstorrent/codeowner-bypass
 tests/nightly/t3000/ @tenstorrent/codeowner-bypass @tenstorrent/metalium-developers-ops-data-movement
 tests/nightly/tg/ @tenstorrent/codeowner-bypass @tenstorrent/metalium-developers-ops-data-movement

.github/actions/analyze-workflow-data/owners.json

@@ -574,10 +574,6 @@
     { "job-name-component": "Galaxy Wan2.2 demo tests", "owner": { "id": "U03FJB5TM5Y", "name": "Colman Glagovich" } },
     { "job-name-component": "Galaxy Mochi demo tests", "owner": { "id": "U09ELB03XRU", "name": "Stephen Osborne" } },
     { "job-name-component": "Galaxy DeepSeek v3 demo tests", "owner": { "id": "U08H32XUS9W", "name": "Yousef Al Rawwash" } },
-    { "job-name-component": "ops-sanity-tests / example test", "owner": { "id": "U08DEGUJY3H", "name": "Rose Li" } },
-    { "job-name-component": "example test", "owner": { "id": "U08DEGUJY3H", "name": "Rose Li" } },
-    { "job-name-component": "ops-sanity-tests / example test BH", "owner": { "id": "U08DEGUJY3H", "name": "Rose Li" } },
-    { "job-name-component": "example test BH", "owner": { "id": "U08DEGUJY3H", "name": "Rose Li" } },
     { "job-name-component": "t3000-demo-tests / t3k sd35 large tests", "owner": { "id": "U03FJB5TM5Y", "name": "Colman Glagovich" } },
     { "job-name-component": "t3k sd35 large tests", "owner": { "id": "U03FJB5TM5Y", "name": "Colman Glagovich" } },
     { "job-name-component": "t3000-demo-tests / t3k gpt oss tests", "owner": { "id": "U08TJ70UFRT", "name": "Harry Andrews" } },

.github/actions/analyze-workflow-data/update-owners-from-pipeline.js

@@ -7,7 +7,6 @@ const path = require('path');
  * Examples:
  *   t3k_unit_tests.yaml -> t3000-unit-tests
  *   t3k_demo_tests.yaml -> t3000-demo-tests
- *   ops/sanity/tests.yaml -> ops-sanity
  */
 function getWorkflowPrefix(filePath) {
   const relativePath = path.relative(path.join(__dirname, '../../..', 'tests/pipeline_reorg'), filePath);

.github/actions/sweep-run-analysis/scripts/send_slack_notification.py

@@ -8,7 +8,6 @@
 import os
 import sys
 import time
-from typing import Optional
 
 import requests
 
@@ -341,10 +340,15 @@ def build_cancelled_block() -> list[dict]:
     ]
 
 
-def build_superset_link_block(run_id: Optional[int]) -> dict:
-    """Build the Superset dashboard link block."""
-    if run_id:
-        url = f"{SUPERSET_BASE_URL}?run_id={run_id}"
+def build_superset_link_block() -> dict:
+    """Build the Superset dashboard link block.
+
+    Uses gh_run_number (GITHUB_RUN_ID) since it's available immediately,
+    whereas run_id (database PK) requires Airflow ingestion first.
+    """
+    if GITHUB_RUN_ID:
+        # Use GitHub run ID - works immediately, dashboard shows data after ingestion
+        url = f"{SUPERSET_BASE_URL}?gh_run_number={GITHUB_RUN_ID}"
         text = f"<{url}|View in Superset>"
     else:
         text = f"<{GITHUB_ACTIONS_URL}|View in GitHub Actions>"
@@ -419,7 +423,7 @@ def build_slack_message(results: dict, conclusion: str) -> dict:
 
     # Superset link
     blocks.append({"type": "divider"})
-    blocks.append(build_superset_link_block(results.get("run_id")))
+    blocks.append(build_superset_link_block())
 
     return {"blocks": blocks}
 

.github/scripts/utils/prepare_test_matrix.py

@@ -89,15 +89,21 @@ def load_tests(tests_yaml_path):
 
 def build_test_matrix(tests, enabled_skus, sku_config):
     """
-    Filter tests based on enabled SKUs and add runs_on labels.
+    Filter tests based on enabled SKUs and expand multi-SKU entries into flat matrix entries.
+
+    Each test entry may define multiple SKUs in its 'skus' dict. This function
+    expands each test into one matrix entry per enabled SKU, with the appropriate
+    timeout and runs_on labels.
 
     Args:
-        tests: List of test dictionaries
+        tests: List of test dictionaries (with 'skus' dict)
         enabled_skus: List of enabled SKU strings
         sku_config: Dictionary mapping SKU names to their configuration
 
     Returns:
-        Filtered list of test dictionaries with runs_on added
+        Filtered list of flat test dictionaries. Each entry has all keys from the
+        test (e.g. name, cmd, model, owner_id, team) with skus removed and sku,
+        timeout, and runs_on set for the selected SKU.
     """
     if not enabled_skus:
         print("::error::No SKUs enabled. At least one SKU must be specified.")
@@ -112,25 +118,40 @@ def build_test_matrix(tests, enabled_skus, sku_config):
     filtered_tests = []
 
     for test in tests:
-        test_sku = test.get("sku")
         test_name = test.get("name", "Unnamed Test")
+        test_skus = test.get("skus")
 
-        # Skip tests without a SKU
-        if not test_sku:
-            print(f"::warning::Test '{test_name}' has no SKU, skipping")
+        # Skip tests without skus
+        if not test_skus or not isinstance(test_skus, dict):
+            print(f"::warning::Test '{test_name}' has no valid 'skus' mapping, skipping")
             continue
 
-        # Filter: only include tests whose SKU is in the enabled list
-        if test_sku in enabled_skus:
-            # Add runs_on from SKU config
-            if test_sku in sku_config:
-                test_with_runs_on = test.copy()
-                test_with_runs_on["runs_on"] = sku_config[test_sku].get("runs_on", [])
-                filtered_tests.append(test_with_runs_on)
-            else:
-                print(f"::warning::SKU '{test_sku}' for test '{test_name}' not found in SKU config, skipping")
+        # Determine which of this test's SKUs are enabled
+        matching_skus = [s for s in test_skus if s in enabled_skus]
+
+        # Append SKU to name when the same test runs on more than one SKU
+        append_sku_to_name = len(matching_skus) > 1
+
+        for sku_name in matching_skus:
+            sku_test_config = test_skus[sku_name]
+
+            if sku_name not in sku_config:
+                print(f"::warning::SKU '{sku_name}' for test '{test_name}' not found in SKU config, skipping")
                 continue
 
+            # Start from test copy so all keys (model, arch, etc.) are preserved
+            entry = test.copy()
+            entry.pop("skus", None)
+            entry["sku"] = sku_name
+            entry["timeout"] = sku_test_config.get("timeout", 0)
+            entry["runs_on"] = sku_config[sku_name].get("runs_on", [])
+            if append_sku_to_name:
+                entry["name"] = f"{test_name} [{sku_name}]"
+            for key, value in sku_test_config.items():
+                if key != "timeout" and value is not None:
+                    entry[key] = value
+            filtered_tests.append(entry)
+
     if not filtered_tests:
         print(f"::error::No tests selected for enabled SKUs '{','.join(enabled_skus)}'. Failing pipeline.")
         sys.exit(1)

.github/scripts/utils/verify_time_budget.py

@@ -30,7 +30,7 @@ def verify_timeouts(tests_file, time_budget_file, workflow_name):
         test_name = test.get("name", "Unnamed Test")
 
         # Validate that all mandatory keys exist for this test
-        required_keys = ["timeout", "sku", "team"]
+        required_keys = ["skus", "team"]
         missing_keys = [key for key in required_keys if key not in test]
         if missing_keys:
             print(
@@ -39,14 +39,29 @@ def verify_timeouts(tests_file, time_budget_file, workflow_name):
             errors_found = True
             continue  # Skip this invalid test
 
-        test_timeout = test["timeout"]
-        test_sku = test["sku"]
+        test_skus = test["skus"]
         test_team = test["team"]
 
-        # Use a tuple (team, sku) as the key for summation
-        budget_key = (test_team, test_sku)
-        budget_totals[budget_key] += test_timeout
-        print(f"  Test '{test_name}' (Team: {test_team}, SKU: {test_sku}) adds {test_timeout} min.")
+        if not isinstance(test_skus, dict) or not test_skus:
+            print(
+                f"  [ERROR] Validation FAILED! Test '{test_name}' has invalid 'skus' field. "
+                f"Expected a non-empty mapping of SKU names to their config."
+            )
+            errors_found = True
+            continue
+
+        for sku_name, sku_config in test_skus.items():
+            if not isinstance(sku_config, dict) or "timeout" not in sku_config:
+                print(f"  [ERROR] Validation FAILED! Test '{test_name}', SKU '{sku_name}' is missing 'timeout'.")
+                errors_found = True
+                continue
+
+            test_timeout = sku_config["timeout"]
+
+            # Use a tuple (team, sku) as the key for summation
+            budget_key = (test_team, sku_name)
+            budget_totals[budget_key] += test_timeout
+            print(f"  Test '{test_name}' (Team: {test_team}, SKU: {sku_name}) adds {test_timeout} min.")
 
     if errors_found:
         print(f"\nMissing keys in {tests_file}. Please fix the entries above.")

.github/sku_config.yaml

@@ -4,6 +4,26 @@
 # Each SKU entry contains:
 #   runs_on: Array of GitHub Actions runner labels that tests for this SKU.
 skus:
+  wh_llmbox:
+    runs_on:
+      - arch-wormhole_b0
+      - config-t3000
+      - pipeline-functional
+      - in-service
+
+  wh_llmbox_perf:
+    runs_on:
+      - arch-wormhole_b0
+      - config-t3000
+      - pipeline-perf
+      - in-service
+
+  bh_p100:
+    runs_on:
+      - cloud-virtual-machine
+      - P100
+      - in-service
+
   wh_galaxy:
     runs_on:
       - arch-wormhole_b0
@@ -17,3 +37,19 @@ skus:
       - pipeline-functional
       - topology-6u
       - in-service
+
+  wh_n150_civ2:
+    runs_on:
+      - tt-ubuntu-2204-N150-viommu-stable
+
+  wh_n300_civ2:
+    runs_on:
+      - tt-ubuntu-2204-N300-viommu-stable
+
+  wh_llmbox_civ2:
+    runs_on:
+      - tt-ubuntu-2204-N300-llmbox-stable
+
+  bh_p150b_civ2:
+    runs_on:
+      - tt-ubuntu-2204-P150b-stable

.github/time_budget.yaml

@@ -97,14 +97,14 @@ scaleout:
 
 models:
   e2e:
-    wh_llmbox: 50
+    wh_llmbox: 110
   unit:
-    wh_llmbox: 287
+    wh_llmbox: 292
   integration:
     wh_llmbox: 422
   perf:
-    wh_llmbox: 685
+    wh_llmbox_perf: 685
     wh_galaxy: 260
   demo:
-    wh_llmbox: 1090
+    wh_llmbox_perf: 1090
     wh_galaxy: 270

.github/workflows/_auto-retry-post-commit.yaml

@@ -19,6 +19,7 @@ on:
       # to want to to
       - "zzz TG Quick tests"
       - "apc nightly debug run"
+      - "apc nightly debug run with watcher"
     types:
       - completed
 

.github/workflows/_produce-data.yaml

@@ -37,6 +37,7 @@ on:
       - "All post-commit tests"
       - "PR - All post-commit tests"
       - "apc nightly debug run"
+      - "apc nightly debug run with watcher"
       - "(Single-card) Model perf tests"
       - "(Single-card) Device perf regressions"
       - "(Single-card) Demo tests"