PR1: Move metric.py to scripts #171
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: validate-model | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| branches: main | |
| jobs: | |
| cml: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: iterative/setup-cml@v2 | |
| - uses: astral-sh/setup-uv@v1 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version-file: ".python-version" | |
| - name: Generate datasets and models | |
| run: | | |
| # Supervised game: Generate datasets and models | |
| uv run proteingym-base list-datasets datasets | jq 'map(select(.name == "charge_ladder" or .name == "NEIME_2019")) | map({name: .name, input_filename: .input_filename}) | . = {"datasets": .}' > benchmark/supervised/local/datasets.json | |
| uv run proteingym-base list-models models | jq 'map(select(.name == "pls")) | map({name: .name, input_filename: .input_filename}) | . = {"models": .}' > benchmark/supervised/local/models.json | |
| # Zero-shot game: Generate datasets and models | |
| uv run proteingym-base list-datasets datasets | jq 'map(select(.name == "ranganathan")) | map({name: .name, input_filename: .input_filename}) | . = {"datasets": .}' > benchmark/zero_shot/local/datasets.json | |
| uv run proteingym-base list-models models | jq 'map(select(.name == "esm")) | map({name: .name, input_filename: .input_filename}) | . = {"models": .}' > benchmark/zero_shot/local/models.json | |
| - name: Run model validation | |
| env: | |
| repo_token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} | |
| run: | | |
| # Supervised game: Run DVC repro | |
| uv run dvc repro benchmark/supervised/local/dvc.yaml | |
| # Zero-shot game: Run DVC repro | |
| uv run dvc repro benchmark/zero_shot/local/dvc.yaml | |
| # Create report with metrics | |
| # Pick one supervised model | |
| echo "✅ Supervised models have all passed validation." >> report.md | |
| echo '```csv' >> report.md | |
| cat ./benchmark/supervised/local/metric/charge_ladder_pls.csv >> report.md | |
| echo '```' >> report.md | |
| # Pick one zero-shot model | |
| echo "✅ Zero-shot models have all passed validation." >> report.md | |
| echo '```csv' >> report.md | |
| cat ./benchmark/zero_shot/local/metric/ranganathan_esm.csv >> report.md | |
| echo '```' >> report.md | |
| # Send comment to PR | |
| echo "Send comment to PR" | |
| cml comment create report.md |