Fix tests introduced in 1.1 and add conformance testing (#706) #75
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Cromwell Spec Conformance - WDL 1.1 | |
| on: | |
| push: | |
| jobs: | |
| conformance: | |
| runs-on: ubuntu-latest | |
| name: Run WDL 1.1 Spec Conformance Tests | |
| steps: | |
| - name: Checkout WDL spec repo | |
| uses: actions/checkout@v4 | |
| - name: Install Java (for Cromwell) | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y openjdk-11-jre curl | |
| - name: Download latest Cromwell | |
| run: | | |
| CROMWELL_URL=$(curl -s https://api.github.com/repos/broadinstitute/cromwell/releases/latest \ | |
| | grep "browser_download_url" \ | |
| | grep "cromwell-.*\.jar" \ | |
| | cut -d '"' -f 4) | |
| echo "Downloading $CROMWELL_URL" | |
| curl -L $CROMWELL_URL -o cromwell.jar | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| - name: Install dependencies | |
| run: | | |
| pip install subby | |
| - name: Clone WDL tests | |
| run: | | |
| git clone https://github.com/openwdl/wdl-tests.git | |
| - name: Extract tests | |
| run: | | |
| python3 wdl-tests/scripts/extract_tests.py \ | |
| -i ${{ github.workspace }}/SPEC.md \ | |
| -d ${{ github.workspace }}/tests/data/ \ | |
| -O cromwell_test -v 1.1 | |
| find cromwell_test/ -name "*.wdl" -type f -exec sed -i 's/^version 1.1/version development/' {} + | |
| - name: Run tests and capture results | |
| run: | | |
| mkdir -p cromwell_tests/artifacts | |
| echo "📊 Running tests..." | |
| python3 wdl-tests/scripts/run_tests_cromwell.py \ | |
| -T ${{ github.workspace }}/cromwell_test \ | |
| -c ${{ github.workspace }}/cromwell_test/test_config.json \ | |
| -D ${{ github.workspace }}/cromwell_test/data \ | |
| --cromwell-jar ${{ github.workspace }}/cromwell.jar \ | |
| -O cromwell_results | tee cromwell_tests/artifacts/result.log | |
| echo "📊 Parsing test summary..." | |
| total=$(grep "Total tests:" cromwell_tests/artifacts/result.log | awk '{print $3}') | |
| passed=$(grep "Passed:" cromwell_tests/artifacts/result.log | awk '{print $2}') | |
| warnings=$(grep "Warnings:" cromwell_tests/artifacts/result.log | awk '{print $2}') | |
| failed=$(grep "Failures:" cromwell_tests/artifacts/result.log | awk '{print $2}') | |
| invalid=$(grep "Invalid outputs:" cromwell_tests/artifacts/result.log | awk '{print $3}') | |
| ignored=$(grep "Ignored:" cromwell_tests/artifacts/result.log | awk '{print $2}') | |
| # Write summary JSON | |
| cat <<EOF > cromwell_tests/artifacts/results.json | |
| { | |
| "total": $total, | |
| "passed": $passed, | |
| "warnings": $warnings, | |
| "failures": $failed, | |
| "invalid_outputs": $invalid, | |
| "ignored": $ignored | |
| } | |
| EOF | |
| # Copy list of failures to artifacts | |
| cp cromwell_tests/failed_tests.txt cromwell_tests/artifacts/failed_tests.txt || touch cromwell_tests/artifacts/failed_tests.txt | |
| # Generate shields.io badge JSON | |
| color="green" | |
| total_run=$((total - ignored)) | |
| if [ "$passed" -lt "$total_run" ]; then color="yellow"; fi | |
| if [ "$passed" -eq 0 ]; then color="red"; fi | |
| cat <<EOF > cromwell_tests/artifacts/shields.json | |
| { | |
| "schemaVersion": 1, | |
| "label": "Cromwell WDL 1.1", | |
| "message": "$passed/$total_run passed", | |
| "color": "$color" | |
| } | |
| EOF | |
| - name: Upload test artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: cromwell-test-results | |
| path: cromwell_tests/artifacts/ | |
| - name: Publish badge JSON to current branch | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| git config --global user.email "github-actions[bot]@users.noreply.github.com" | |
| git config --global user.name "github-actions[bot]" | |
| BRANCH="${GITHUB_REF_NAME}" | |
| RUN_ID="${{ github.run_id }}" | |
| REPO="${{ github.repository }}" | |
| RUN_URL="https://github.com/$REPO/actions/runs/$RUN_ID" | |
| REPO_URL="https://x-access-token:${GITHUB_TOKEN}@github.com/${{ github.repository }}.git" | |
| git fetch origin $BRANCH | |
| git checkout $BRANCH | |
| mkdir -p shields | |
| cp cromwell_tests/artifacts/shields.json shields/cromwell_shields.json | |
| git add shields/cromwell_shields.json | |
| # Update README only if needed | |
| sed_pattern="https://raw.githubusercontent.com/openwdl/wdl/.*/shields/cromwell_shields.json" | |
| new_url="https://raw.githubusercontent.com/openwdl/wdl/$BRANCH/shields/cromwell_shields.json" | |
| run_pattern="https://github.com/$REPO/actions/runs/[0-9]+" | |
| new_run="https://github.com/$REPO/actions/runs/$RUN_ID" | |
| if grep -qE "$sed_pattern" README.md; then | |
| git pull | |
| sed -E -i "/cromwell_shields\.json/s|$sed_pattern|$new_url|g" README.md | |
| sed -E -i "/cromwell_shields\.json/s|$run_pattern|$new_run|g" README.md | |
| git add README.md | |
| fi | |
| # Only commit if there are staged changes | |
| if ! git diff --cached --quiet; then | |
| git commit -m "chore: updates Cromwell shields badge for \`$BRANCH\`" | |
| git push $REPO_URL $BRANCH | |
| else | |
| echo "🟢 No changes to commit" | |
| fi |