[CICD] Upload unittest coverage report to FlagCICD platform && Access FlagCICD runner

.github/configs/cuda.yml

@@ -8,18 +8,18 @@ display_name: 'NVIDIA CUDA (A100)'
 ci_image: harbor.baai.ac.cn/flagscale/cuda12.8.1-torch2.7.1-python3.10-te2.9:20260209
 
 # Runner labels for self-hosted A100 node
+# runner_labels:
+# - self-hosted
+# - Linux
+# - X64
+# - nvidia
+# - gpu-8
 runner_labels:
-  - self-hosted
-  - Linux
-  - X64
-  - nvidia
-  - gpu-8
+  - nv-8g-cicd-te
 
 # Container volumes
 container_volumes:
   - /home/flagscale_cicd/flask/static:/workspace/report
-  # - .:/opt/transformerengine
-  # - ./ci_logs:/logs
   # - /home/flagscale_cicd/data:/opt/data
 
 # Container options

.github/configs/metax.yml

@@ -6,13 +6,16 @@ hardware_name: metax
 display_name: 'Metax Tests'
 
 ci_image: localhost:5000/megatron-lm-with-te:v1
+# ci_image: harbor.baai.ac.cn/flagscale/megatron-lm-with-te:202603231839
 
 runner_labels:
   - self-hosted
   - Linux
   - X64
   - metax
   - dev
+# runner_labels:
+#   - mx-4g-cicd-te
 
 container_volumes:
   - /nfs/metax_fs:/nfs/metax_fs

.github/workflows/all_tests_common.yml

@@ -92,13 +92,13 @@ jobs:
     uses: ./.github/workflows/unit_tests_common.yml
     name: unit_tests
     with:
-      setup_commands: ${{ inputs.setup_commands }}
       platform: ${{ inputs.platform }}
       device: ${{ matrix.device }}
       image: ${{ needs.checkout_and_config.outputs.ci_image }}
       runs_on: ${{ needs.checkout_and_config.outputs.runs_on }}
       container_volumes: ${{ needs.checkout_and_config.outputs.container_volumes }}
       container_options: ${{ needs.checkout_and_config.outputs.container_options }}
+      setup_commands: ${{ inputs.setup_commands }}
       ignored_tests: ${{ needs.checkout_and_config.outputs.ignored_tests }}
       build_env: ${{ needs.checkout_and_config.outputs.build_env }}
 

.github/workflows/all_tests_cuda.yml

@@ -1,10 +1,10 @@
 name: cuda_tests
 
 on:
-  # push:
-  #   branches: ["main"]
-  # pull_request:
-  #   branches: ["main"]
+  push:
+    branches: ["main"]
+  pull_request:
+    branches: ["main"]
   workflow_dispatch:
 
 concurrency:

.github/workflows/qa-l0-te-cpp-unittest-pytorch-lint.yml

@@ -72,7 +72,7 @@ jobs:
 
           # Install Python dependencies with version pinning
           echo "=== Installing Python Dependencies ==="
-          pip install transformers expecttest
+          pip install transformers expecttest nvdlfw-inspect --quiet
 
           # Build and install transformer_engine with verbose output
           echo "=== Building & Installing Transformer Engine ==="

.github/workflows/unit_tests_common.yml

@@ -115,23 +115,46 @@ jobs:
           # For debugging, you can force this to true
           echo "should_run=true" >> $GITHUB_OUTPUT
 
-      - name: Checkout Source Code
-        if: steps.should_run.outputs.should_run == 'true'
+      # Cuda requires git safe.directory configuration and 3 checkout attempts to handle submodule-heavy repos
+      - name: Configure Git Safe Directory on Cuda
+        if: steps.should_run.outputs.should_run == 'true' && inputs.platform == 'cuda'
+        run: /usr/bin/git config --global safe.directory '*'
+
+      - name: Checkout Source Code on Cuda (attempt 1)
+        id: checkout1
+        if: steps.should_run.outputs.should_run == 'true' && inputs.platform == 'cuda'
         uses: actions/checkout@v4
+        continue-on-error: true
         with:
+          fetch-depth: 0
+          submodules: recursive
           set-safe-directory: true
 
-      # - name: Activate Python environment
-      #   run: |
-      #     if [[ "$PLATFORM" == "cuda" ]] && [ -f /opt/miniconda3/etc/profile.d/conda.sh ]; then
-      #       source /opt/miniconda3/etc/profile.d/conda.sh
-      #       conda activate flagscale-train
-      #     elif [ -f /opt/conda/etc/profile.d/conda.sh ]; then
-      #       source /opt/conda/etc/profile.d/conda.sh
-      #       conda activate base
-      #     fi
-      #     echo "PATH=$PATH" >> $GITHUB_ENV
-      #     echo "Python: $(which python3) ($(python3 --version 2>&1))"
+      - name: Checkout Source Code on Cuda (attempt 2)
+        id: checkout2
+        if: steps.should_run.outputs.should_run == 'true' && inputs.platform == 'cuda' && steps.checkout1.outcome == 'failure'
+        uses: actions/checkout@v4
+        continue-on-error: true
+        with:
+          fetch-depth: 0
+          submodules: recursive
+          set-safe-directory: true
+
+      - name: Checkout Source Code on Cuda (attempt 3)
+        id: checkout3
+        if: steps.should_run.outputs.should_run == 'true' && inputs.platform == 'cuda' && steps.checkout2.outcome == 'failure'
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: recursive
+          set-safe-directory: true
+
+      # Metax no need submodules
+      - name: Checkout Source Code on Metax
+        if: steps.should_run.outputs.should_run == 'true' && inputs.platform == 'metax'
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
       - name: Environment Setup on Cuda
         if: steps.should_run.outputs.should_run == 'true' && inputs.platform == 'cuda'
@@ -149,8 +172,10 @@ jobs:
 
           echo "===== Step 2: Build & Install TransformerEngine ====="
           cd $GITHUB_WORKSPACE
-          pip install nvdlfw-inspect --no-deps
-          pip install --no-build-isolation . -v --no-deps
+
+          pip install nvdlfw-inspect --quiet
+          pip install expecttest --quiet
+          pip install . -v --no-deps --no-build-isolation
 
           echo "===== Step 3: Verify Installation ====="
           python3 tests/pytorch/test_sanity_import.py
@@ -186,7 +211,8 @@ jobs:
           echo "===== Step 4: Remove Existing TransformerEngine ====="
           # Prevent conflicts with preinstalled or incompatible versions
           python3 -m pip uninstall transformer_engine -y || true
-          python3 -m pip install nvdlfw-inspect --no-deps || true
+          python3 -m pip install nvdlfw-inspect --quiet
+          python3 -m pip install expecttest --quiet
 
           # echo "===== Step 5: Install Metax Binary Backend ====="
           # # Install prebuilt Metax backend (required for MACA operators)
@@ -229,7 +255,6 @@ jobs:
         working-directory: ${{ github.workspace }}
         run: |
           set -euo pipefail
-          ${{ inputs.setup_commands }}
 
           # Load platform-specific environment variables
           while IFS='=' read -r key value; do
@@ -257,11 +282,15 @@ jobs:
           # Ensure log directory exists regardless of volume mount state
           mkdir -p /logs
 
-          # Enable coverage collection for all pytest invocations in test.sh
-          # PYTEST_ADDOPTS is automatically appended to every pytest call
-          if [ "${{ inputs.upload_coverage }}" = "true" ]; then
-            pip3 install pytest-cov 2>/dev/null || true
-            export PYTEST_ADDOPTS="--cov=transformer_engine --cov-append --cov-report="
+          # Coverage setup: install once + configure collection via PYTEST_ADDOPTS
+          COVERAGE_ENABLED=false
+          if [ "${{ inputs.upload_coverage }}" = "true" ] && [ "${{ matrix.test_group.test_type }}" = "unittest" ]; then
+            if pip3 install coverage pytest-cov --quiet 2>/dev/null; then
+              export PYTEST_ADDOPTS="--cov=transformer_engine --cov-append --cov-report="
+              COVERAGE_ENABLED=true
+            else
+              echo "WARNING: Failed to install coverage/pytest-cov, coverage collection disabled"
+            fi
           fi
 
           if [[ "${{ matrix.test_group.name }}" == *"lint"* ]]; then
@@ -274,55 +303,39 @@ jobs:
           fi
 
           bash ${{ matrix.test_group.path }}
-        timeout-minutes: 60
+          exit_code=$?
+
+          # Combine coverage fragments and generate JSON report
+          if [ "$COVERAGE_ENABLED" = "true" ]; then
+            python3 -m coverage combine --keep 2>/dev/null || true
+            python3 -m coverage json \
+              -o "coverage-${{ inputs.platform }}-${{ inputs.device }}-${{ matrix.test_group.name }}.json" \
+              --include="transformer_engine/*" 2>/dev/null \
+              || echo "WARNING: No coverage data found"
+          fi
 
-      - name: Generate Coverage Report
-        if: inputs.upload_coverage && matrix.test_group.test_type == 'unittest'
-        working-directory: ${{ github.workspace }}
-        env:
-          PLATFORM: ${{ inputs.platform }}
-          DEVICE: ${{ inputs.device }}
-        run: |
-          # Install coverage (may already be present)
-          pip3 install coverage pytest-cov 2>/dev/null || true
-
-          # Merge all .coverage* files produced by sub-processes (torchrun spawns workers)
-          python3 -m coverage combine --keep 2>/dev/null || true
-          # Generate JSON coverage report (requires .coverage data from pytest --cov)
-          python3 -m coverage json -o "coverage-${PLATFORM}-${DEVICE}.json" \
-            --include="transformer_engine/*" 2>/dev/null || echo "WARNING: No coverage data found, skipping coverage-${PLATFORM}-${DEVICE}.json"
-        continue-on-error: true
+          exit $exit_code
+        timeout-minutes: 60
 
       - name: Upload Coverage Report
         if: inputs.upload_coverage && matrix.test_group.test_type == 'unittest'
         uses: actions/upload-artifact@v4
         continue-on-error: true
         with:
-          name: coverage-${{ inputs.platform }}-${{ inputs.device }}
+          name: coverage-${{ inputs.platform }}-${{ inputs.device }}-${{ matrix.test_group.name }}
           path: |
-            coverage-${{ inputs.platform }}-${{ inputs.device }}.json
-
-      - name: Check FlagCICD Reachability
-        if: inputs.upload_coverage && matrix.test_group.test_type == 'unittest'
-        id: check_flagcicd
-        continue-on-error: true
-        run: |
-          if curl -sf --max-time 3 --connect-timeout 2 \
-               "http://flagcicd-inner.flagos.net:8000/" -o /dev/null 2>/dev/null; then
-            echo "reachable=true" >> $GITHUB_OUTPUT
-          else
-            echo "reachable=false" >> $GITHUB_OUTPUT
-            echo "INFO: flagcicd-inner.flagos.net unreachable from this runner, skipping report upload"
-          fi
+            coverage-${{ inputs.platform }}-${{ inputs.device }}-${{ matrix.test_group.name }}.json
 
       - name: Upload Coverage Report to FlagCICD
-        if: inputs.upload_coverage && matrix.test_group.test_type == 'unittest' && steps.check_flagcicd.outputs.reachable == 'true'
+        if: inputs.upload_coverage && matrix.test_group.test_type == 'unittest'
         uses: flagos-ai/FlagOps/actions/post-pytest-report@v2
         continue-on-error: true
+        env:
+          NO_PROXY: "flagcicd-inner.flagos.net"
         with:
           backend_url: 'http://flagcicd-inner.flagos.net:8000/metrics/'
           user_id: '000000000000000000'
-          report_path: 'coverage-${{ inputs.platform }}-${{ inputs.device }}.json'
+          report_path: 'coverage-${{ inputs.platform }}-${{ inputs.device }}-${{ matrix.test_group.name }}.json'
           fail_on_error: 'false'
 
       # - name: Debug - keep container alive on failure

qa/L0_pytorch_debug_unittest/test.sh

@@ -18,8 +18,6 @@ FAIL=0
 
 # It is not installed as a requirement,
 # because it is not available on PyPI.
-pip uninstall -y nvdlfw-inspect
-pip install git+https://github.com/NVIDIA/nvidia-dlfw-inspect.git
 pip install pytest==8.2.1
 
 run_test_step() {

qa/L0_pytorch_lint/test.sh

@@ -6,7 +6,7 @@ set -e
 
 : "${TE_PATH:=/opt/transformerengine}"
 
-pip3 install cpplint==1.6.0 pylint==3.3.1
+pip3 install cpplint==1.6.0 pylint==3.3.4
 if [ -z "${PYTHON_ONLY}" ]
 then
   cd $TE_PATH