recover setup_commands for unittests

Author: BrianPei

.github/workflows/all_tests_common.yml

@@ -7,6 +7,10 @@ on:
         required: true
         type: string
         description: Platform name (e.g., cuda, default)
+      setup_commands:
+        required: false
+        type: string
+        default: ''
 
 jobs:
   checkout_and_config:
@@ -88,6 +92,7 @@ jobs:
     uses: ./.github/workflows/unit_tests_common.yml
     name: unit_tests
     with:
+      setup_commands: ${{ inputs.setup_commands }}
       platform: ${{ inputs.platform }}
       device: ${{ matrix.device }}
       image: ${{ needs.checkout_and_config.outputs.ci_image }}

.github/workflows/unit_tests_common.yml

@@ -26,6 +26,11 @@ on:
         required: false
         type: string
         default: ''
+      # New input for hardware-specific initialization (e.g., conda activate)
+      setup_commands:
+        required: false
+        type: string
+        default: ''
       # Platform-specific build environment variables (JSON object from config)
       build_env:
         required: false
@@ -115,6 +120,18 @@ jobs:
         uses: actions/checkout@v4
         with:
           set-safe-directory: true
+
+      # - name: Activate Python environment
+      #   run: |
+      #     if [[ "$PLATFORM" == "cuda" ]] && [ -f /opt/miniconda3/etc/profile.d/conda.sh ]; then
+      #       source /opt/miniconda3/etc/profile.d/conda.sh
+      #       conda activate flagscale-train
+      #     elif [ -f /opt/conda/etc/profile.d/conda.sh ]; then
+      #       source /opt/conda/etc/profile.d/conda.sh
+      #       conda activate base
+      #     fi
+      #     echo "PATH=$PATH" >> $GITHUB_ENV
+      #     echo "Python: $(which python3) ($(python3 --version 2>&1))"
 
       - name: Environment Setup on Cuda
         if: steps.should_run.outputs.should_run == 'true' && inputs.platform == 'cuda'
@@ -212,6 +229,7 @@ jobs:
         working-directory: ${{ github.workspace }}
         run: |
           set -euo pipefail
+          ${{ inputs.setup_commands }}
 
           # Load platform-specific environment variables
           while IFS='=' read -r key value; do
@@ -239,12 +257,11 @@ jobs:
           # Ensure log directory exists regardless of volume mount state
           mkdir -p /logs
 
-          # Coverage setup: install once + configure collection via PYTEST_ADDOPTS
-          COVERAGE_ENABLED=false
-          if [ "${{ inputs.upload_coverage }}" = "true" ] && [ "${{ matrix.test_group.test_type }}" = "unittest" ]; then
-            pip3 install coverage pytest-cov --quiet 2>/dev/null || true
+          # Enable coverage collection for all pytest invocations in test.sh
+          # PYTEST_ADDOPTS is automatically appended to every pytest call
+          if [ "${{ inputs.upload_coverage }}" = "true" ]; then
+            pip3 install pytest-cov 2>/dev/null || true
             export PYTEST_ADDOPTS="--cov=transformer_engine --cov-append --cov-report="
-            COVERAGE_ENABLED=true
           fi
 
           if [[ "${{ matrix.test_group.name }}" == *"lint"* ]]; then
@@ -257,39 +274,55 @@ jobs:
           fi
 
           bash ${{ matrix.test_group.path }}
-          exit_code=$?
-
-          # Combine coverage fragments and generate JSON report
-          if [ "$COVERAGE_ENABLED" = "true" ]; then
-            python3 -m coverage combine --keep 2>/dev/null || true
-            python3 -m coverage json \
-              -o "coverage-${{ inputs.platform }}-${{ inputs.device }}-${{ matrix.test_group.name }}.json" \
-              --include="transformer_engine/*" 2>/dev/null \
-              || echo "WARNING: No coverage data found"
-          fi
-
-          exit $exit_code
         timeout-minutes: 60
 
+      - name: Generate Coverage Report
+        if: inputs.upload_coverage && matrix.test_group.test_type == 'unittest'
+        working-directory: ${{ github.workspace }}
+        env:
+          PLATFORM: ${{ inputs.platform }}
+          DEVICE: ${{ inputs.device }}
+        run: |
+          # Install coverage (may already be present)
+          pip3 install coverage pytest-cov 2>/dev/null || true
+
+          # Merge all .coverage* files produced by sub-processes (torchrun spawns workers)
+          python3 -m coverage combine --keep 2>/dev/null || true
+          # Generate JSON coverage report (requires .coverage data from pytest --cov)
+          python3 -m coverage json -o "coverage-${PLATFORM}-${DEVICE}.json" \
+            --include="transformer_engine/*" 2>/dev/null || echo "WARNING: No coverage data found, skipping coverage-${PLATFORM}-${DEVICE}.json"
+        continue-on-error: true
+
       - name: Upload Coverage Report
         if: inputs.upload_coverage && matrix.test_group.test_type == 'unittest'
         uses: actions/upload-artifact@v4
         continue-on-error: true
         with:
-          name: coverage-${{ inputs.platform }}-${{ inputs.device }}-${{ matrix.test_group.name }}
+          name: coverage-${{ inputs.platform }}-${{ inputs.device }}
           path: |
-            coverage-${{ inputs.platform }}-${{ inputs.device }}-${{ matrix.test_group.name }}.json
+            coverage-${{ inputs.platform }}-${{ inputs.device }}.json
 
-      - name: Upload Coverage Report to FlagCICD
+      - name: Check FlagCICD Reachability
         if: inputs.upload_coverage && matrix.test_group.test_type == 'unittest'
+        id: check_flagcicd
+        continue-on-error: true
+        run: |
+          if curl -sf --max-time 3 --connect-timeout 2 \
+               "http://flagcicd-inner.flagos.net:8000/" -o /dev/null 2>/dev/null; then
+            echo "reachable=true" >> $GITHUB_OUTPUT
+          else
+            echo "reachable=false" >> $GITHUB_OUTPUT
+            echo "INFO: flagcicd-inner.flagos.net unreachable from this runner, skipping report upload"
+          fi
+
+      - name: Upload Coverage Report to FlagCICD
+        if: inputs.upload_coverage && matrix.test_group.test_type == 'unittest' && steps.check_flagcicd.outputs.reachable == 'true'
         uses: flagos-ai/FlagOps/actions/post-pytest-report@v2
         continue-on-error: true
-        env:
-          NO_PROXY: "flagcicd-inner.flagos.net"
         with:
           backend_url: 'http://flagcicd-inner.flagos.net:8000/metrics/'
           user_id: '000000000000000000'
-          report_path: 'coverage-${{ inputs.platform }}-${{ inputs.device }}-${{ matrix.test_group.name }}.json'
+          report_path: 'coverage-${{ inputs.platform }}-${{ inputs.device }}.json'
           fail_on_error: 'false'
 
       # - name: Debug - keep container alive on failure