Skip to content

Commit fc516cf

Browse files
tintinrevienttintinrevient
authored
Merge pull request #117 from ProteinGym/feat/create-benchmark
Create benchmark from datasets and models JSON files
2 parents 20f9946 + a323868 commit fc516cf

34 files changed

+1840
-4646
lines changed

‎.github/workflows/cml.yaml‎

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -21,21 +21,26 @@ jobs:
2121
with:
2222
python-version-file: ".python-version"
2323

24+
- name: Generate datasets and models
25+
run: |
26+
# Supervised game: Generate datasets and models
27+
uv run proteingym-base list-datasets datasets | jq 'map(select(.name == "charge_ladder" or .name == "NEIME_2019")) | map({name: .name, input_filename: .input_filename})' | yq -p json -o json '. = {"datasets": .}' > benchmark/supervised/local/datasets.json
28+
uv run proteingym-base list-models models | jq 'map(select(.name == "pls")) | map({name: .name, input_filename: .input_filename})' | yq -p json -o json '. = {"models": .}' > benchmark/supervised/local/models.json
29+
30+
# Zero-shot game: Generate datasets and models
31+
uv run proteingym-base list-datasets datasets | jq 'map(select(.name == "ranganathan")) | map({name: .name, input_filename: .input_filename})' | yq -p json -o json '. = {"datasets": .}' > benchmark/zero_shot/local/datasets.json
32+
uv run proteingym-base list-models models | jq 'map(select(.name == "esm")) | map({name: .name, input_filename: .input_filename})' | yq -p json -o json '. = {"models": .}' > benchmark/zero_shot/local/models.json
33+
2434
- name: Run model validation
2535
env:
2636
repo_token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
27-
run: |
28-
# Configure PAT to git clone the private repo: proteingym-base
29-
git config --global credential.helper store
30-
echo "https://${{ secrets.USERNAME }}:${{ secrets.PERSONAL_ACCESS_TOKEN }}@github.com" > ~/.git-credentials
31-
uv sync
32-
33-
# Run DVC repro
34-
echo "https://${{ secrets.USERNAME }}:${{ secrets.PERSONAL_ACCESS_TOKEN }}@github.com" > benchmark/supervised/git-auth.txt
35-
echo "https://${{ secrets.USERNAME }}:${{ secrets.PERSONAL_ACCESS_TOKEN }}@github.com" > benchmark/zero_shot/git-auth.txt
37+
run: |
38+
# Supervised game: Run DVC repro
3639
uv run dvc repro benchmark/supervised/local/dvc.yaml
40+
41+
# Zero-shot game: Run DVC repro
3742
uv run dvc repro benchmark/zero_shot/local/dvc.yaml
38-
43+
3944
# Create report with metrics
4045
# Pick one supervised model
4146
echo "✅ Supervised models have all passed validation." >> report.md

‎benchmark/supervised/local/default.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
output:
2+
prediction: output
3+
metric: metric

‎benchmark/supervised/local/dvc.lock‎

Lines changed: 0 additions & 97 deletions
This file was deleted.

‎benchmark/supervised/local/dvc.yaml‎

Lines changed: 17 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,57 +1,42 @@
11
vars:
2-
- params.yaml
3-
4-
- datasets:
5-
- name: charge_ladder
6-
container_path: /datasets/dummy/dataset.zip
7-
local_path: ../../../datasets/dummy/dataset.zip
8-
9-
- name: neime
10-
container_path: /datasets/neime/dataset.zip
11-
local_path: ../../../datasets/neime/dataset.zip
12-
13-
- models:
14-
- name: pls
15-
container_path: /models/pls/README.md
16-
local_path: ../../../models/pls/README.md
17-
dockerfile: ../../../models/pls/Dockerfile
2+
- default.yaml
3+
- datasets.json
4+
- models.json
185

196
stages:
207

218
setup:
22-
cmd:
23-
- mkdir -p logs ${destination.output_dir} ${destination.metric_dir}
9+
cmd:
10+
- mkdir -p logs ${output.prediction} ${output.metric}
2411
- echo "Created local directories" > logs/setup.txt
2512
outs:
2613
- logs/setup.txt
27-
14+
2815
create_training_job:
29-
matrix:
16+
matrix:
3017
dataset: ${datasets}
3118
model: ${models}
3219

33-
cmd:
34-
- docker build --build-arg GIT_CACHE_BUST=${git.git_cache_bust} --secret id=git_auth,src=../git-auth.txt -f ${item.model.dockerfile} -t ${item.model.name}:latest ../../..
35-
- docker run --rm -v $(realpath ${source.datasets_dir}):/datasets -v $(realpath ${source.models_dir}):/models -v $(realpath ${destination.output_dir}):/opt/ml/model ${item.model.name}:latest train --dataset-file ${item.dataset.container_path} --model-card-file ${item.model.container_path}
20+
cmd:
21+
- docker build -f $(dirname ${item.model.input_filename})/Dockerfile -t ${item.model.name}:latest $(dirname ${item.model.input_filename})
22+
- docker run --rm -v $(realpath ${item.dataset.input_filename}):/$(basename ${item.dataset.input_filename}) -v $(realpath ${item.model.input_filename}):/$(basename ${item.model.input_filename}) -v $(realpath ${output.prediction}):/opt/ml/model ${item.model.name}:latest train --dataset-file /$(basename ${item.dataset.input_filename}) --model-card-file /$(basename ${item.model.input_filename})
3623
- docker image prune -a -f
37-
3824
deps:
3925
- logs/setup.txt
40-
- ${item.dataset.local_path}
41-
- ${item.model.local_path}
42-
- ${item.model.dockerfile}
26+
- ${item.dataset.input_filename}
27+
- ${item.model.input_filename}
4328
outs:
44-
- ${destination.output_dir}/${item.dataset.name}_${item.model.name}.csv:
29+
- ${output.prediction}/${item.dataset.name}_${item.model.name}.csv:
4530
cache: true
4631

4732
calculate_metric:
4833
matrix:
4934
dataset: ${datasets}
5035
model: ${models}
5136

52-
cmd: uv run proteingym-benchmark metric calc --output-path ${destination.output_dir}/${item.dataset.name}_${item.model.name}.csv --metric-path ${destination.metric_dir}/${item.dataset.name}_${item.model.name}.csv
37+
cmd: uv run proteingym-benchmark metric calc --output-path ${output.prediction}/${item.dataset.name}_${item.model.name}.csv --metric-path ${output.metric}/${item.dataset.name}_${item.model.name}.csv
5338
deps:
54-
- ${destination.output_dir}/${item.dataset.name}_${item.model.name}.csv
39+
- ${output.prediction}/${item.dataset.name}_${item.model.name}.csv
5540
outs:
56-
- ${destination.metric_dir}/${item.dataset.name}_${item.model.name}.csv:
57-
cache: true
41+
- ${output.metric}/${item.dataset.name}_${item.model.name}.csv:
42+
cache: true

‎benchmark/supervised/local/params.yaml‎

Lines changed: 0 additions & 10 deletions
This file was deleted.

‎benchmark/zero_shot/local/default.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
output:
2+
prediction: output
3+
metric: metric

‎benchmark/zero_shot/local/dvc.lock‎

Lines changed: 0 additions & 54 deletions
This file was deleted.

‎benchmark/zero_shot/local/dvc.yaml‎

Lines changed: 18 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,53 +1,42 @@
11
vars:
2-
- params.yaml
3-
4-
- datasets:
5-
- name: ranganathan
6-
container_path: /datasets/ranganathan/dataset.zip
7-
local_path: ../../../datasets/ranganathan/dataset.zip
8-
9-
- models:
10-
- name: esm
11-
container_path: /models/esm/README.md
12-
local_path: ../../../models/esm/README.md
13-
dockerfile: ../../../models/esm/Dockerfile
2+
- default.yaml
3+
- datasets.json
4+
- models.json
145

156
stages:
167

178
setup:
18-
cmd:
19-
- mkdir -p logs ${destination.output_dir} ${destination.metric_dir}
9+
cmd:
10+
- mkdir -p logs ${output.prediction} ${output.metric}
2011
- echo "Created local directories" > logs/setup.txt
2112
outs:
2213
- logs/setup.txt
2314

2415
create_training_job:
25-
matrix:
16+
matrix:
2617
dataset: ${datasets}
2718
model: ${models}
28-
29-
cmd:
30-
- docker build --build-arg GIT_CACHE_BUST=${git.git_cache_bust} --secret id=git_auth,src=../git-auth.txt -f ${item.model.dockerfile} -t ${item.model.name}:latest ../../..
31-
- docker run --rm -v $(realpath ${source.datasets_dir}):/datasets -v $(realpath ${source.models_dir}):/models -v $(realpath ${destination.output_dir}):/opt/ml/model ${item.model.name}:latest train --dataset-file ${item.dataset.container_path} --model-card-file ${item.model.container_path}
32-
- docker image prune -a -f
3319

20+
cmd:
21+
- docker build -f $(dirname ${item.model.input_filename})/Dockerfile -t ${item.model.name}:latest $(dirname ${item.model.input_filename})
22+
- docker run --rm -v $(realpath ${item.dataset.input_filename}):/$(basename ${item.dataset.input_filename}) -v $(realpath ${item.model.input_filename}):/$(basename ${item.model.input_filename}) -v $(realpath ${output.prediction}):/opt/ml/model ${item.model.name}:latest train --dataset-file /$(basename ${item.dataset.input_filename}) --model-card-file /$(basename ${item.model.input_filename})
23+
- docker image prune -a -f
3424
deps:
3525
- logs/setup.txt
36-
- ${item.dataset.local_path}
37-
- ${item.model.local_path}
38-
- ${item.model.dockerfile}
26+
- ${item.dataset.input_filename}
27+
- ${item.model.input_filename}
3928
outs:
40-
- ${destination.output_dir}/${item.dataset.name}_${item.model.name}.csv:
29+
- ${output.prediction}/${item.dataset.name}_${item.model.name}.csv:
4130
cache: true
4231

4332
calculate_metric:
4433
matrix:
4534
dataset: ${datasets}
4635
model: ${models}
47-
48-
cmd: uv run proteingym-benchmark metric calc --output-path ${destination.output_dir}/${item.dataset.name}_${item.model.name}.csv --metric-path ${destination.metric_dir}/${item.dataset.name}_${item.model.name}.csv
36+
37+
cmd: uv run proteingym-benchmark metric calc --output-path ${output.prediction}/${item.dataset.name}_${item.model.name}.csv --metric-path ${output.metric}/${item.dataset.name}_${item.model.name}.csv
4938
deps:
50-
- ${destination.output_dir}/${item.dataset.name}_${item.model.name}.csv
39+
- ${output.prediction}/${item.dataset.name}_${item.model.name}.csv
5140
outs:
52-
- ${destination.metric_dir}/${item.dataset.name}_${item.model.name}.csv:
53-
cache: true
41+
- ${output.metric}/${item.dataset.name}_${item.model.name}.csv:
42+
cache: true

‎benchmark/zero_shot/local/params.yaml‎

Lines changed: 0 additions & 10 deletions
This file was deleted.

‎datasets/NEIME_2019.pgdata‎

1.27 MB
Binary file not shown.

0 commit comments

Comments
 (0)