conformance.yml

name: Conformance Tests

# This workflow runs conformance tests in parallel to reduce CI time

on:
  push:
    branches: [ main, develop ]
  pull_request:
    branches: [ main, develop ]
  workflow_dispatch:  # Allow manual triggering

# Set permissions for GITHUB_TOKEN
permissions:
  contents: read

jobs:
  # Run standard library and integration tests first (fast, basic checks)
  basic-tests:
    runs-on: ubuntu-latest
    
    steps:
      - name: Checkout repository
        uses: actions/checkout@v7
      
      - name: Setup Rust toolchain
        uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          toolchain: stable
      
      - name: Run standard library tests
        run: cargo test --features ci --lib
      
      - name: Run integration tests
        run: cargo test --features ci --test core --test extensions --test validation --test infrastructure
  
  # Setup test suites once for all conformance jobs
  setup-test-suites:
    runs-on: ubuntu-latest
    
    steps:
      - name: Cache test suites
        id: cache-test-suites
        uses: actions/cache@v6
        with:
          path: test_suites
          key: ${{ runner.os }}-test-suites-${{ hashFiles('.github/workflows/conformance.yml') }}
          restore-keys: |
            ${{ runner.os }}-test-suites-
      
      - name: Clone 3MF test suites
        run: |
          if [ ! -d "test_suites" ]; then
            git clone --depth 1 https://github.com/3MFConsortium/test_suites.git
          fi
      
      - name: Upload test suites artifact
        uses: actions/upload-artifact@v7
        with:
          name: test-suites
          path: test_suites/
          retention-days: 1
  
  # Run each conformance suite in parallel
  conformance:
    runs-on: ubuntu-latest
    needs: [basic-tests, setup-test-suites]
    
    strategy:
      fail-fast: false
      matrix:
        suite:
          # Suites 1-8 use lowercase directory names
          - name: suite1_core_slice_prod
            dir: suite1_core_slice_prod
            pos_dir: positive_test_cases
            neg_dir: negative_test_cases
          - name: suite2_core_prod_matl
            dir: suite2_core_prod_matl
            pos_dir: positive_test_cases
            neg_dir: negative_test_cases
          - name: suite3_core
            dir: suite3_core
            pos_dir: positive_test_cases
            neg_dir: negative_test_cases
          - name: suite4_core_slice
            dir: suite4_core_slice
            pos_dir: positive_test_cases
            neg_dir: negative_test_cases
          - name: suite5_core_prod
            dir: suite5_core_prod
            pos_dir: positive_test_cases
            neg_dir: negative_test_cases
          - name: suite6_core_matl
            dir: suite6_core_matl
            pos_dir: positive_test_cases
            neg_dir: negative_test_cases
          - name: suite7_beam
            dir: suite7_beam
            pos_dir: positive_test_cases
            neg_dir: negative_test_cases
          - name: suite8_secure
            dir: suite8_secure
            pos_dir: positive_test_cases
            neg_dir: negative_test_cases
          # Suites 9-11 use title case with spaces (as defined in upstream test suite repo)
          - name: suite9_core_ext
            dir: suite9_core_ext
            pos_dir: Positive Tests
            neg_dir: Negative Tests
          - name: suite10_boolean
            dir: suite10_boolean
            pos_dir: Positive Tests
            neg_dir: Negative Tests
          - name: suite11_displacement
            dir: suite11_Displacement
            pos_dir: Positive Tests
            neg_dir: Negative Tests
    
    steps:
      - name: Checkout repository
        uses: actions/checkout@v7
      
      - name: Setup Rust toolchain
        uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          toolchain: stable
      
      - name: Prepare test_suites directory
        run: mkdir -p test_suites

      - name: Download test suites artifact
        uses: actions/download-artifact@v8.0.1
        with:
          name: test-suites
          path: test_suites
      
      - name: Run conformance tests for ${{ matrix.suite.name }}
        id: run-tests
        run: |
          echo "Testing suite: ${{ matrix.suite.name }}"
          if [ "${{ matrix.suite.name }}" = "suite8_secure" ]; then
            cargo test --features ci,crypto --test conformance ${{ matrix.suite.name }} -- --nocapture 2>&1 | tee test-output.txt
            exit ${PIPESTATUS[0]}
          else
            cargo test --features ci --test conformance ${{ matrix.suite.name }} -- --nocapture 2>&1 | tee test-output.txt
            exit ${PIPESTATUS[0]}
          fi
      
      - name: Generate Job Summary
        if: always()
        shell: bash
        run: |
          SUITE="${{ matrix.suite.name }}"
          case "$SUITE" in
            suite1_core_slice_prod) DESC="Core + Production + Slice Extensions" ;;
            suite2_core_prod_matl)  DESC="Core + Production + Materials Extensions" ;;
            suite3_core)            DESC="Core Specification (Basic)" ;;
            suite4_core_slice)      DESC="Core + Slice Extension" ;;
            suite5_core_prod)       DESC="Core + Production Extension" ;;
            suite6_core_matl)       DESC="Core + Materials Extension" ;;
            suite7_beam)            DESC="Beam Lattice Extension" ;;
            suite8_secure)          DESC="Secure Content Extension" ;;
            suite9_core_ext)        DESC="Core Extensions" ;;
            suite10_boolean)        DESC="Boolean Operations Extension" ;;
            suite11_displacement)   DESC="Displacement Extension" ;;
            *)                      DESC="$SUITE" ;;
          esac

          if [ -f test-output.txt ]; then
            POS_LINE=$(grep "^${SUITE}: .* positive tests" test-output.txt || true)
            NEG_LINE=$(grep "^${SUITE}: .* negative tests" test-output.txt || true)
            HAS_FAILURE=$(grep -cE "FAILED|failed tests" test-output.txt || true)
          else
            POS_LINE=""
            NEG_LINE=""
            HAS_FAILURE=1
          fi

          if [ "${HAS_FAILURE}" = "0" ]; then
            STATUS="✅"
          else
            STATUS="❌"
          fi

          {
            echo "## ${STATUS} ${SUITE}"
            echo "*${DESC}*"
            echo ""
            [ -n "$POS_LINE" ] && echo "- **${POS_LINE}**"
            [ -n "$NEG_LINE" ] && echo "- **${NEG_LINE}**"

            if [ -f test-output.txt ] && grep -q "Failed tests:" test-output.txt; then
              FAIL_COUNT=$(grep -cE "^  [0-9]+\." test-output.txt || echo "?")
              echo ""
              echo "<details>"
              echo "<summary>❌ Failed test cases (${FAIL_COUNT})</summary>"
              echo ""
              echo '```'
              sed -n '/Failed tests:/,/^test result:/p' test-output.txt \
                | grep -v "^test result:" \
                | head -100
              echo '```'
              echo "</details>"
            fi
            echo ""
          } >> "$GITHUB_STEP_SUMMARY"
  
  # Generate overall summary after all suites complete
  conformance-summary:
    runs-on: ubuntu-latest
    needs: [conformance, setup-test-suites]
    if: always()
    
    steps:
      - name: Checkout repository
        uses: actions/checkout@v7
      
      - name: Setup Rust toolchain
        uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          toolchain: stable
      
      - name: Prepare directories
        run: mkdir -p test_suites suite-reports

      - name: Download test suites artifact
        uses: actions/download-artifact@v8.0.1
        with:
          name: test-suites
          path: test_suites
      
      - name: Generate overall conformance summary
        run: |
          echo "# Conformance Test Results" > conformance-summary.md
          echo "" >> conformance-summary.md
          echo "## Overall Summary" >> conformance-summary.md
          echo "Tests were run in parallel in the conformance job. See individual suite reports below." >> conformance-summary.md
        continue-on-error: true
      
      - name: Download all suite reports
        uses: actions/download-artifact@v8.0.1
        with:
          pattern: conformance-report-*
          path: suite-reports
      
      - name: Generate CONFORMANCE_REPORT.md
        run: |
          echo "Generating basic conformance report..."
          TIMESTAMP=$(date -u '+%Y-%m-%d %H:%M:%S UTC')
          cat > CONFORMANCE_REPORT.md << 'EOFMARKER'
          # 3MF Conformance Test Report

          **Generated:** TIMESTAMP_PLACEHOLDER

          ## Test Execution

          Conformance tests were executed in parallel across 11 test suites.
          See individual suite reports in the GitHub Actions artifacts for detailed results.

          ## Test Suites

          The following test suites were executed:
          - suite1_core_slice_prod: Core + Production + Slice Extensions
          - suite2_core_prod_matl: Core + Production + Materials Extensions
          - suite3_core: Core Specification (Basic)
          - suite4_core_slice: Core + Slice Extension
          - suite5_core_prod: Core + Production Extension
          - suite6_core_matl: Core + Materials Extension
          - suite7_beam: Beam Lattice Extension
          - suite8_secure: Secure Content Extension
          - suite9_core_ext: Core Extensions
          - suite10_boolean: Boolean Operations Extension
          - suite11_Displacement: Displacement Extension

          ---

          ## About This Report

          This report is automatically generated by the CI workflow.
          Tests are run against the official 3MF Consortium test cases from [3MFConsortium/test_suites](https://github.com/3MFConsortium/test_suites).

          **Test Methodology:**
          - **Positive tests** validate that valid 3MF files parse successfully
          - **Negative tests** validate that invalid 3MF files are properly rejected

          **To generate a detailed report locally:**
          ```bash
          ./generate_conformance_report.sh
          ```
          EOFMARKER
          sed -i "s/TIMESTAMP_PLACEHOLDER/$TIMESTAMP/" CONFORMANCE_REPORT.md
          echo "✅ Basic conformance report generated"
      
      - name: Upload combined conformance report
        uses: actions/upload-artifact@v7
        with:
          name: conformance-report-combined
          path: |
            conformance-summary.md
            suite-reports/
            CONFORMANCE_REPORT.md
          retention-days: 30