Exclude metadata.json from pytest report selection

psanal35 · psanal35 · commit 418847f93f4f · 2026-02-06T21:26:11.000Z
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
@@ -1,57 +1,88 @@
 name: Nightly
 
 on:
-  schedule:
-    - cron: "0 1 * * *"
   workflow_dispatch:
-  pull_request:
-    paths:
-      - '.github/workflows/nightly.yml'
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-permissions:
-  contents: read
-  packages: write
+    inputs:
+      source-repo:
+        description: "ORG/REPO"
+        required: false
+        default: "rocm/jax"
+        type: string
+      source-github-run-id:
+        description: "Repo GitHub Run ID"
+        required: false
+        default: ""
+        type: string
+    secrets:
+      ROCM_JAX_DB_HOSTNAME:
+        required: true
+      ROCM_JAX_DB_USERNAME:
+        required: true
+      ROCM_JAX_DB_PASSWORD:
+        required: true
+      ROCM_JAX_DB_NAME:
+        required: true
 
 jobs:
-  call-build-wheels:
-    strategy:
-      fail-fast: false
-      matrix:
-        rocm-version: ["7.1.1", "7.2.0"]
-        include:
-          - rocm-version: "7.1.1"
-            runner-label: '["linux-x86-64-1gpu-amd"]'
-          - rocm-version: "7.2.0"
-            runner-label: '["linux-x86-64-1gpu-amd"]'
-    uses: ./.github/workflows/build-wheels.yml
-    with:
-      python-versions: "3.11,3.12"
-      rocm-version: ${{ matrix.rocm-version }}
-      rocm-build-job: ${{ matrix.rocm-build-job }}
-      rocm-build-num: ${{ matrix.rocm-build-num }}
-      runner-label: ${{ matrix.runner-label }}
-    secrets:
-      rbe_ci_cert: ${{ secrets.RBE_CI_CERT }}
-      rbe_ci_key: ${{ secrets.RBE_CI_KEY }}
-  call-build-docker:
-    needs: call-build-wheels
-    strategy:
-      fail-fast: false
-      matrix:
-        rocm-version: ["7.1.1", "7.2.0"]
-        include:
-          - rocm-version: "7.1.1"
-            runner-label: '["linux-x86-64-1gpu-amd"]'
-          - rocm-version: "7.2.0"
-            runner-label: '["linux-x86-64-1gpu-amd"]'
-    uses: ./.github/workflows/build-docker.yml
-    with:
-      rocm-version: ${{ matrix.rocm-version }}
-      rocm-build-job: ${{ matrix.rocm-build-job }}
-      rocm-build-num: ${{ matrix.rocm-build-num }}
-      runner-label: ${{ matrix.runner-label }}
-      extra-cr-tag: "nightly"
+  download-logs:
+    runs-on: linux-mi355-1
+    steps:
+      - name: Download logs artifact from source run
+        env:
+          GH_TOKEN: ${{ secrets.SECRET_TOKEN }}
+        run: |
+          gh run download "${{ inputs.source-github-run-id }}" \
+            -R "${{ inputs.source-repo }}" \
+            -D "logs-${{ inputs.source-github-run-id }}"
+      - name: Upload logs for DB job
+        uses: actions/upload-artifact@v4
+        with:
+          name: logs-${{ inputs.source-github-run-id }}
+          path: logs-${{ inputs.source-github-run-id }}/
+  upload-to-db:
+    needs: download-logs
+    runs-on: mysqldb
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v4
+      - name: Download logs from workflow artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: logs-${{ inputs.source-github-run-id }}
+          path: logs-${{ inputs.source-github-run-id }}
+      - name: Upload logs to MySQL database
+        env:
+          ROCM_JAX_DB_HOSTNAME: ${{ secrets.ROCM_JAX_DB_HOSTNAME }}
+          ROCM_JAX_DB_USERNAME: ${{ secrets.ROCM_JAX_DB_USERNAME }}
+          ROCM_JAX_DB_PASSWORD: ${{ secrets.ROCM_JAX_DB_PASSWORD }}
+          ROCM_JAX_DB_NAME: ${{ secrets.ROCM_JAX_DB_NAME }}
+          ROCM_VERSION: ${{ inputs.rocm-version }}
+          UBUNTU_VERSION: ${{ inputs.ubuntu-version }}
+          LOGS_DIR: logs-${{ inputs.source-github-run-id }}
+          GITHUB_SHA: ${{ inputs.github-sha }}
+          GITHUB_RUN_ID: ${{ inputs.github-run-id }}
+          ROCM_BUILD_NUM: ${{ inputs.rocm-build-num }}
+        run: |
+          python3 -m venv venv
+          source venv/bin/activate
+          pip install --upgrade pip
+          pip install mysql-connector-python
+
+          for artifact_dir in "$LOGS_DIR"/*; do
+            [ -d "$artifact_dir" ] || continue
+            meta="$artifact_dir/metadata.json"
+            [ -f "$meta" ] || continue
+            RUNNER_LABEL=$(python -c "import json;print(json.load(open('$meta'))['runner_label'])")
+            UBUNTU_VERSION=$(python -c "import json;print(json.load(open('$meta'))['ubuntu_version'])")
+            ROCM_VERSION=$(python -c "import json;print(json.load(open('$meta'))['rocm_version'])")
+            GITHUB_SHA=$(python -c "import json;print(json.load(open('$meta'))['github_sha'])")
+            GITHUB_RUN_ID=$(python -c "import json;print(json.load(open('$meta'))['github_run_id'])")
+            python ci/upload_test_to_db.py \
+              --logs_dir "$artifact_dir" \
+              --run-tag "ci-run" \
+              --runner-label "$RUNNER_LABEL" \
+              --ubuntu-version "$UBUNTU_VERSION" \
+              --rocm-version "${ROCM_VERSION//.}" \
+              --commit-sha "$GITHUB_SHA" \
+              --github-run-id "$GITHUB_RUN_ID"
+          done
diff --git a/.github/workflows/pytest-results-to-db.yml b/.github/workflows/pytest-results-to-db.yml
@@ -25,16 +25,8 @@ on:
 
 jobs:
   download-logs:
-    runs-on: linux-x86-64-1gpu-amd
+    runs-on: linux-mi355-1
     steps:
-      - name: Install GitHub CLI
-        run: |
-          type -p curl >/dev/null || (apt update && apt install curl -y)
-          curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg
-          chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg
-          echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null
-          apt update
-          apt install gh -y
       - name: Download logs artifact from source run
         env:
           GH_TOKEN: ${{ secrets.SECRET_TOKEN }}
diff --git a/ci/upload_test_to_db.py b/ci/upload_test_to_db.py
@@ -2,9 +2,9 @@
 """
 Upload pytest results to MySQL.
 Tables:
- - ci_runs_dup:    one row per run
- - ci_tests_dup:   one row per unique test
- - ci_results_dup: one row per test per run
+ - ci_runs:    one row per run
+ - ci_tests:   one row per unique test
+ - ci_results: one row per test per run
 """
 
 from __future__ import annotations
@@ -78,6 +78,7 @@ def find_single_report_json(logs_dir: Path) -> Path:
         for p in logs_dir.iterdir()
         if p.is_file()
         and p.suffix.lower() == ".json"
+        and p.name not in {"metadata.json"}
         and not p.name.endswith("last_running.json")
     ]
     if not jsons:
@@ -267,10 +268,10 @@ def connect():
 
 
 def insert_run(cur, created_at: datetime, meta: dict) -> int:
-    """Insert one row into ci_runs_dup and return run_id. Idempotence is not enforced here."""
+    """Insert one row into ci_runs and return run_id. Idempotence is not enforced here."""
     cur.execute(
         """
-       INSERT INTO ci_runs_dup (
+       INSERT INTO ci_runs (
            created_at, commit_sha, runner_label, ubuntu_version,
            rocm_version, build_num, github_run_id, run_tag, logs_path
        ) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)
@@ -291,11 +292,11 @@ def insert_run(cur, created_at: datetime, meta: dict) -> int:
 
 
 def sync_tests_and_get_ids(cur, tests: List[dict]) -> Dict[Tuple[str, str, str], int]:
-    """Ensure all tests exist in ci_tests_dup and return an ID mapping.
+    """Ensure all tests exist in ci_tests and return an ID mapping.
 
     Uses a TEMPORARY TABLE for efficiency with large runs:
       1) Bulk insert unique (filename, classname, test_name) into a temp table.
-      2) INSERT any missing rows into ci_tests_dup in one set operation.
+      2) INSERT any missing rows into ci_tests in one set operation.
       3) SELECT back (file, class, test) -> id mapping in one query.
     """
     uniq = {nodeid_parts(t["nodeid"]) for t in tests}
@@ -319,10 +320,10 @@ def sync_tests_and_get_ids(cur, tests: List[dict]) -> Dict[Tuple[str, str, str],
 
     cur.execute(
         """
-       INSERT INTO ci_tests_dup (filename, classname, test_name)
+       INSERT INTO ci_tests (filename, classname, test_name)
        SELECT s.filename, s.classname, s.test_name
        FROM tmp_tests s
-       LEFT JOIN ci_tests_dup t
+       LEFT JOIN ci_tests t
          ON t.filename = s.filename
         AND t.classname = s.classname
         AND t.test_name = s.test_name
@@ -334,7 +335,7 @@ def sync_tests_and_get_ids(cur, tests: List[dict]) -> Dict[Tuple[str, str, str],
         """
        SELECT t.id, s.filename, s.classname, s.test_name
        FROM tmp_tests s
-       JOIN ci_tests_dup t
+       JOIN ci_tests t
          ON t.filename = s.filename
         AND t.classname = s.classname
         AND t.test_name = s.test_name
@@ -360,7 +361,7 @@ def batch_insert_results(
         return
 
     sql = """
-       INSERT INTO ci_results_dup
+       INSERT INTO ci_results
            (run_id, test_id, outcome, duration, longrepr, message, skip_label)
        VALUES (%s,%s,%s,%s,%s,%s,%s)
        ON DUPLICATE KEY UPDATE
@@ -392,9 +393,9 @@ def upload_pytest_results(  # pylint: disable=too-many-arguments, too-many-local
 
     Flow:
       1) Parse JSONs and gather tests.
-      2) Insert a ci_runs_dup row and get run_id.
+      2) Insert a ci_runs row and get run_id.
       3) Ensure all tests exist; get (file,class,test) -> test_id map.
-      4) Bulk insert/update ci_results_dup for this run.
+      4) Bulk insert/update ci_results for this run.
     """
     report = find_single_report_json(logs_dir)
     created_at, tests = load_from_single_json(report)
