Skip to content

Commit 3f813e1

Browse files
vsmalladivsmalladi
authored
Fix tests introduced in 1.1 and add conformance testing (#706)
* Fix issues with examples Closes #707, Closes #708, Closes #701. Closes #702 * Fix examples for returnCodes to be lower camelcase. Closes #710. * Remove Advanced Task Examples to clearly distinguish what is the testing is in scope. Closes #730 * Add in CI/CD for Miniwdl, Sprocket, Toil and Cromwell for spec compliance
1 parent 3a6f9c0 commit 3f813e1

File tree

11 files changed

+643
-243
lines changed

11 files changed

+643
-243
lines changed

‎.github/workflows/cromwell-test.yml‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
name: Cromwell Spec Conformance - WDL 1.1
2+
3+
on:
4+
push:
5+
6+
jobs:
7+
conformance:
8+
runs-on: ubuntu-latest
9+
name: Run WDL 1.1 Spec Conformance Tests
10+
11+
steps:
12+
- name: Checkout WDL spec repo
13+
uses: actions/checkout@v4
14+
15+
- name: Install Java (for Cromwell)
16+
run: |
17+
sudo apt-get update
18+
sudo apt-get install -y openjdk-11-jre curl
19+
20+
- name: Download latest Cromwell
21+
run: |
22+
CROMWELL_URL=$(curl -s https://api.github.com/repos/broadinstitute/cromwell/releases/latest \
23+
| grep "browser_download_url" \
24+
| grep "cromwell-.*\.jar" \
25+
| cut -d '"' -f 4)
26+
echo "Downloading $CROMWELL_URL"
27+
curl -L $CROMWELL_URL -o cromwell.jar
28+
29+
- name: Set up Python
30+
uses: actions/setup-python@v5
31+
with:
32+
python-version: '3.10'
33+
34+
- name: Install dependencies
35+
run: |
36+
pip install subby
37+
38+
- name: Clone WDL tests
39+
run: |
40+
git clone https://github.com/openwdl/wdl-tests.git
41+
42+
- name: Extract tests
43+
run: |
44+
python3 wdl-tests/scripts/extract_tests.py \
45+
-i ${{ github.workspace }}/SPEC.md \
46+
-d ${{ github.workspace }}/tests/data/ \
47+
-O cromwell_test -v 1.1
48+
find cromwell_test/ -name "*.wdl" -type f -exec sed -i 's/^version 1.1/version development/' {} +
49+
50+
- name: Run tests and capture results
51+
run: |
52+
mkdir -p cromwell_tests/artifacts
53+
54+
echo "📊 Running tests..."
55+
python3 wdl-tests/scripts/run_tests_cromwell.py \
56+
-T ${{ github.workspace }}/cromwell_test \
57+
-c ${{ github.workspace }}/cromwell_test/test_config.json \
58+
-D ${{ github.workspace }}/cromwell_test/data \
59+
--cromwell-jar ${{ github.workspace }}/cromwell.jar \
60+
-O cromwell_results | tee cromwell_tests/artifacts/result.log
61+
62+
echo "📊 Parsing test summary..."
63+
64+
total=$(grep "Total tests:" cromwell_tests/artifacts/result.log | awk '{print $3}')
65+
passed=$(grep "Passed:" cromwell_tests/artifacts/result.log | awk '{print $2}')
66+
warnings=$(grep "Warnings:" cromwell_tests/artifacts/result.log | awk '{print $2}')
67+
failed=$(grep "Failures:" cromwell_tests/artifacts/result.log | awk '{print $2}')
68+
invalid=$(grep "Invalid outputs:" cromwell_tests/artifacts/result.log | awk '{print $3}')
69+
ignored=$(grep "Ignored:" cromwell_tests/artifacts/result.log | awk '{print $2}')
70+
71+
# Write summary JSON
72+
cat <<EOF > cromwell_tests/artifacts/results.json
73+
{
74+
"total": $total,
75+
"passed": $passed,
76+
"warnings": $warnings,
77+
"failures": $failed,
78+
"invalid_outputs": $invalid,
79+
"ignored": $ignored
80+
}
81+
EOF
82+
83+
# Copy list of failures to artifacts
84+
cp cromwell_tests/failed_tests.txt cromwell_tests/artifacts/failed_tests.txt || touch cromwell_tests/artifacts/failed_tests.txt
85+
86+
# Generate shields.io badge JSON
87+
color="green"
88+
total_run=$((total - ignored))
89+
if [ "$passed" -lt "$total_run" ]; then color="yellow"; fi
90+
if [ "$passed" -eq 0 ]; then color="red"; fi
91+
cat <<EOF > cromwell_tests/artifacts/shields.json
92+
{
93+
"schemaVersion": 1,
94+
"label": "Cromwell WDL 1.1",
95+
"message": "$passed/$total_run passed",
96+
"color": "$color"
97+
}
98+
EOF
99+
100+
- name: Upload test artifacts
101+
uses: actions/upload-artifact@v4
102+
with:
103+
name: cromwell-test-results
104+
path: cromwell_tests/artifacts/
105+
106+
- name: Publish badge JSON to current branch
107+
env:
108+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
109+
run: |
110+
git config --global user.email "github-actions[bot]@users.noreply.github.com"
111+
git config --global user.name "github-actions[bot]"
112+
BRANCH="${GITHUB_REF_NAME}"
113+
RUN_ID="${{ github.run_id }}"
114+
REPO="${{ github.repository }}"
115+
RUN_URL="https://github.com/$REPO/actions/runs/$RUN_ID"
116+
REPO_URL="https://x-access-token:${GITHUB_TOKEN}@github.com/${{ github.repository }}.git"
117+
118+
git fetch origin $BRANCH
119+
git checkout $BRANCH
120+
121+
mkdir -p shields
122+
cp cromwell_tests/artifacts/shields.json shields/cromwell_shields.json
123+
git add shields/cromwell_shields.json
124+
125+
# Update README only if needed
126+
sed_pattern="https://raw.githubusercontent.com/openwdl/wdl/.*/shields/cromwell_shields.json"
127+
new_url="https://raw.githubusercontent.com/openwdl/wdl/$BRANCH/shields/cromwell_shields.json"
128+
run_pattern="https://github.com/$REPO/actions/runs/[0-9]+"
129+
new_run="https://github.com/$REPO/actions/runs/$RUN_ID"
130+
if grep -qE "$sed_pattern" README.md; then
131+
git pull
132+
sed -E -i "/cromwell_shields\.json/s|$sed_pattern|$new_url|g" README.md
133+
sed -E -i "/cromwell_shields\.json/s|$run_pattern|$new_run|g" README.md
134+
git add README.md
135+
fi
136+
137+
# Only commit if there are staged changes
138+
if ! git diff --cached --quiet; then
139+
git commit -m "chore: updates Cromwell shields badge for \`$BRANCH\`"
140+
git push $REPO_URL $BRANCH
141+
else
142+
echo "🟢 No changes to commit"
143+
fi

‎.github/workflows/miniwdl-test.yml‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
name: MiniWDL Spec Conformance - WDL 1.1
2+
3+
on:
4+
push:
5+
6+
jobs:
7+
conformance:
8+
runs-on: ubuntu-latest
9+
name: Run WDL 1.1 Spec Conformance Tests
10+
11+
steps:
12+
- name: Checkout WDL spec repo
13+
uses: actions/checkout@v4
14+
15+
- name: Set up Python
16+
uses: actions/setup-python@v5
17+
with:
18+
python-version: '3.10'
19+
20+
- name: Install MiniWDL and dependencies
21+
run: |
22+
pip install miniwdl subby
23+
24+
- name: Clone WDL tests
25+
run: |
26+
git clone https://github.com/openwdl/wdl-tests.git
27+
28+
- name: Extract tests
29+
run: |
30+
python wdl-tests/scripts/extract_tests.py -i ${{ github.workspace }}/SPEC.md -d ${{ github.workspace }}/tests/data/ -O miniwdl_test -v 1.1
31+
32+
- name: Run tests and capture results
33+
run: |
34+
mkdir -p miniwdl_tests/artifacts
35+
36+
echo "📊 Running tests..."
37+
python wdl-tests/scripts/run_tests_miniwdl.py -T ${{ github.workspace }}/miniwdl_test -c ${{ github.workspace }}/miniwdl_test/test_config.json -D ${{ github.workspace }}/miniwdl_test/data -O miniwdl_results | tee miniwdl_tests/artifacts/result.log
38+
39+
echo "📊 Parsing test summary..."
40+
41+
total=$(grep "Total tests:" miniwdl_tests/artifacts/result.log | awk '{print $3}')
42+
passed=$(grep "Passed:" miniwdl_tests/artifacts/result.log | awk '{print $2}')
43+
warnings=$(grep "Warnings:" miniwdl_tests/artifacts/result.log | awk '{print $2}')
44+
failed=$(grep "Failures:" miniwdl_tests/artifacts/result.log | awk '{print $2}')
45+
invalid=$(grep "Invalid outputs:" miniwdl_tests/artifacts/result.log | awk '{print $3}')
46+
ignored=$(grep "Ignored:" miniwdl_tests/artifacts/result.log | awk '{print $2}')
47+
48+
# Write summary JSON
49+
cat <<EOF > miniwdl_tests/artifacts/results.json
50+
{
51+
"total": $total,
52+
"passed": $passed,
53+
"warnings": $warnings,
54+
"failures": $failed,
55+
"invalid_outputs": $invalid,
56+
"ignored": $ignored
57+
}
58+
EOF
59+
60+
# Copy list of failures to artifacts
61+
cp miniwdl_test/failed_tests.txt miniwdl_tests/artifacts/failed_tests.txt || touch miniwdl_tests/artifacts/failed_tests.txt
62+
63+
# Generate shields.io badge JSON
64+
color="green"
65+
total_run=$((total - ignored))
66+
if [ "$passed" -lt "$total_run" ]; then color="yellow"; fi
67+
if [ "$passed" -eq 0 ]; then color="red"; fi
68+
cat <<EOF > miniwdl_tests/artifacts/shields.json
69+
{
70+
"schemaVersion": 1,
71+
"label": "MiniWDL WDL 1.1",
72+
"message": "$passed/$total_run passed",
73+
"color": "$color"
74+
}
75+
EOF
76+
77+
- name: Upload test artifacts
78+
uses: actions/upload-artifact@v4
79+
with:
80+
name: miniwdl-test-results
81+
path: miniwdl_tests/artifacts/
82+
83+
- name: Publish badge JSON to current branch
84+
env:
85+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
86+
run: |
87+
git config --global user.email "github-actions[bot]@users.noreply.github.com"
88+
git config --global user.name "github-actions[bot]"
89+
BRANCH="${GITHUB_REF_NAME}"
90+
RUN_ID="${{ github.run_id }}"
91+
REPO="${{ github.repository }}"
92+
RUN_URL="https://github.com/$REPO/actions/runs/$RUN_ID"
93+
REPO_URL="https://x-access-token:${GITHUB_TOKEN}@github.com/${{ github.repository }}.git"
94+
95+
git fetch origin $BRANCH
96+
git checkout $BRANCH
97+
98+
mkdir -p shields
99+
cp miniwdl_tests/artifacts/shields.json shields/miniwdl_shields.json
100+
git add shields/miniwdl_shields.json
101+
102+
# Update README only if needed
103+
sed_pattern="https://raw.githubusercontent.com/openwdl/wdl/.*/shields/miniwdl_shields.json"
104+
new_url="https://raw.githubusercontent.com/openwdl/wdl/$BRANCH/shields/miniwdl_shields.json"
105+
run_pattern="https://github.com/$REPO/actions/runs/[0-9]+"
106+
new_run="https://github.com/$REPO/actions/runs/$RUN_ID"
107+
if grep -qE "$sed_pattern" README.md; then
108+
git pull
109+
sed -E -i "/miniwdl_shields\.json/s|$sed_pattern|$new_url|g" README.md
110+
sed -E -i "/miniwdl_shields\.json/s|$run_pattern|$new_run|g" README.md
111+
git add README.md
112+
fi
113+
114+
115+
# Only commit if there are staged changes
116+
if ! git diff --cached --quiet; then
117+
git commit -m "chore: updates MiniWDL shields badge for \`$BRANCH\`"
118+
git push $REPO_URL $BRANCH
119+
else
120+
echo "🟢 No changes to commit"
121+
fi

‎.github/workflows/sprocket-test.yml‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
name: Sprocket Spec Conformance - WDL 1.1
2+
3+
on:
4+
push:
5+
6+
jobs:
7+
sprocket-conformance:
8+
runs-on: ubuntu-latest
9+
name: Run Sprocket WDL 1.1 Spec Conformance Tests
10+
11+
steps:
12+
- name: Checkout WDL spec repo
13+
uses: actions/checkout@v4
14+
15+
- name: Set up Python
16+
uses: actions/setup-python@v5
17+
with:
18+
python-version: '3.10'
19+
20+
- name: Install Python dependencies
21+
run: |
22+
pip install subby
23+
24+
- name: Install Sprocket dependencies
25+
run: |
26+
curl -y --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
27+
. "$HOME/.cargo/env"
28+
cargo install sprocket --locked
29+
30+
- name: Clone WDL tests
31+
run: |
32+
git clone https://github.com/openwdl/wdl-tests.git
33+
34+
- name: Extract tests
35+
run: |
36+
python wdl-tests/scripts/extract_tests.py -i ${{ github.workspace }}/SPEC.md -d ${{ github.workspace }}/tests/data/ -O sprocket_test -v 1.1
37+
38+
- name: Run tests and capture results
39+
run: |
40+
mkdir -p sprocket_tests/artifacts
41+
42+
echo "📊 Running tests..."
43+
python wdl-tests/scripts/run_tests_sprocket.py -T ${{ github.workspace }}/sprocket_test -c ${{ github.workspace }}/sprocket_test/test_config.json -D ${{ github.workspace }}/sprocket_test/data -O sprocket_results | tee sprocket_tests/artifacts/result.log
44+
45+
echo "📊 Parsing test summary..."
46+
47+
total=$(grep "Total tests:" sprocket_tests/artifacts/result.log | awk '{print $3}')
48+
passed=$(grep "Passed:" sprocket_tests/artifacts/result.log | awk '{print $2}')
49+
warnings=$(grep "Warnings:" sprocket_tests/artifacts/result.log | awk '{print $2}')
50+
failed=$(grep "Failures:" sprocket_tests/artifacts/result.log | awk '{print $2}')
51+
invalid=$(grep "Invalid outputs:" sprocket_tests/artifacts/result.log | awk '{print $3}')
52+
ignored=$(grep "Ignored:" sprocket_tests/artifacts/result.log | awk '{print $2}')
53+
54+
# Write summary JSON
55+
cat <<EOF > sprocket_tests/artifacts/results.json
56+
{
57+
"total": $total,
58+
"passed": $passed,
59+
"warnings": $warnings,
60+
"failures": $failed,
61+
"invalid_outputs": $invalid,
62+
"ignored": $ignored
63+
}
64+
EOF
65+
66+
#Copy list of failures to artifacts
67+
cp sprocket_test/failed_tests.txt sprocket_tests/artifacts/failed_tests.txt || touch sprocket_tests/artifacts/failed_tests.txt
68+
69+
# Generate shields.io badge JSON
70+
color="green"
71+
total_run=$((total - ignored))
72+
if [ "$passed" -lt "$total_run" ]; then color="yellow"; fi
73+
if [ "$passed" -eq 0 ]; then color="red"; fi
74+
cat <<EOF > sprocket_tests/artifacts/shields.json
75+
{
76+
"schemaVersion": 1,
77+
"label": "Sprocket WDL 1.1",
78+
"message": "$passed/$total_run passed",
79+
"color": "$color"
80+
}
81+
EOF
82+
83+
- name: Upload test artifacts
84+
uses: actions/upload-artifact@v4
85+
with:
86+
name: sprocket-test-results
87+
path: sprocket_tests/artifacts/
88+
89+
- name: Publish badge JSON to current branch
90+
env:
91+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
92+
run: |
93+
git config --global user.email "github-actions[bot]@users.noreply.github.com"
94+
git config --global user.name "github-actions[bot]"
95+
BRANCH="${GITHUB_REF_NAME}"
96+
RUN_ID="${{ github.run_id }}"
97+
REPO="${{ github.repository }}"
98+
RUN_URL="https://github.com/$REPO/actions/runs/$RUN_ID"
99+
REPO_URL="https://x-access-token:${GITHUB_TOKEN}@github.com/${{ github.repository }}.git"
100+
101+
git fetch origin $BRANCH
102+
git checkout $BRANCH
103+
104+
mkdir -p shields
105+
cp sprocket_tests/artifacts/shields.json shields/sprocket_shields.json
106+
git add shields/sprocket_shields.json
107+
108+
# Update README only if needed
109+
sed_pattern="https://raw.githubusercontent.com/openwdl/wdl/.*/shields/sprocket_shields.json"
110+
new_url="https://raw.githubusercontent.com/openwdl/wdl/$BRANCH/shields/sprocket_shields.json"
111+
run_pattern="https://github.com/$REPO/actions/runs/[0-9]+"
112+
new_run="https://github.com/$REPO/actions/runs/$RUN_ID"
113+
if grep -qE "$sed_pattern" README.md; then
114+
git pull
115+
sed -E -i "/sprocket_shields.json/s|$sed_pattern|$new_url|g" README.md
116+
sed -E -i "/sprocket_shields.json/s|$run_pattern|$new_run|g" README.md
117+
git add README.md
118+
fi
119+
120+
# Only commit if there are staged changes
121+
if ! git diff --cached --quiet; then
122+
git commit -m "chore: updates Sprocket shields badge for \`$BRANCH\`"
123+
git push $REPO_URL $BRANCH
124+
else
125+
echo "🟢 No changes to commit"
126+
fi

0 commit comments

Comments
 (0)