Create benchmark from datasets and models JSON files

.github/workflows/cml.yaml

@@ -21,21 +21,26 @@ jobs:
         with:
           python-version-file: ".python-version"
 
+      - name: Generate datasets and models
+        run: |
+          # Supervised game: Generate datasets and models
+          uv run proteingym-base list-datasets datasets | jq 'map(select(.name == "charge_ladder" or .name == "NEIME_2019")) | map({name: .name, input_filename: .input_filename})' | yq -p json -o json '. = {"datasets": .}' > benchmark/supervised/local/datasets.json
+          uv run proteingym-base list-models models | jq 'map(select(.name == "pls")) | map({name: .name, input_filename: .input_filename})' | yq -p json -o json '. = {"models": .}' > benchmark/supervised/local/models.json
+
+          # Zero-shot game: Generate datasets and models
+          uv run proteingym-base list-datasets datasets | jq 'map(select(.name == "ranganathan")) | map({name: .name, input_filename: .input_filename})' | yq -p json -o json '. = {"datasets": .}' > benchmark/zero_shot/local/datasets.json
+          uv run proteingym-base list-models models | jq 'map(select(.name == "esm")) | map({name: .name, input_filename: .input_filename})' | yq -p json -o json '. = {"models": .}' > benchmark/zero_shot/local/models.json
+
       - name: Run model validation
         env:
           repo_token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-        run: |
-          # Configure PAT to git clone the private repo: proteingym-base
-          git config --global credential.helper store
-          echo "https://${{ secrets.USERNAME }}:${{ secrets.PERSONAL_ACCESS_TOKEN }}@github.com" > ~/.git-credentials
-          uv sync
-
-          # Run DVC repro
-          echo "https://${{ secrets.USERNAME }}:${{ secrets.PERSONAL_ACCESS_TOKEN }}@github.com" > benchmark/supervised/git-auth.txt
-          echo "https://${{ secrets.USERNAME }}:${{ secrets.PERSONAL_ACCESS_TOKEN }}@github.com" > benchmark/zero_shot/git-auth.txt
+        run: | 
+          # Supervised game: Run DVC repro
           uv run dvc repro benchmark/supervised/local/dvc.yaml
+
+          # Zero-shot game: Run DVC repro
           uv run dvc repro benchmark/zero_shot/local/dvc.yaml
-          
+
           # Create report with metrics
           # Pick one supervised model
           echo "✅ Supervised models have all passed validation." >> report.md

benchmark/supervised/local/default.yaml

@@ -0,0 +1,3 @@
+target:
+  output_path: output
+  metric_path: metric

benchmark/supervised/local/dvc.yaml

@@ -1,57 +1,42 @@
 vars:
-  - params.yaml
-
-  - datasets:
-    - name: charge_ladder
-      container_path: /datasets/dummy/dataset.zip
-      local_path: ../../../datasets/dummy/dataset.zip
-
-    - name: neime
-      container_path: /datasets/neime/dataset.zip
-      local_path: ../../../datasets/neime/dataset.zip
-
-  - models:
-    - name: pls
-      container_path: /models/pls/README.md
-      local_path: ../../../models/pls/README.md
-      dockerfile: ../../../models/pls/Dockerfile
+  - default.yaml
+  - datasets.json
+  - models.json
 
 stages:
 
   setup:
-    cmd: 
-      - mkdir -p logs ${destination.output_dir} ${destination.metric_dir}
+    cmd:
+      - mkdir -p logs ${target.output_path} ${target.metric_path}
       - echo "Created local directories" > logs/setup.txt
     outs:
       - logs/setup.txt
-    
+
   create_training_job:
-    matrix: 
+    matrix:
       dataset: ${datasets}
       model: ${models}
 
-    cmd: 
-      - docker build --build-arg GIT_CACHE_BUST=${git.git_cache_bust} --secret id=git_auth,src=../git-auth.txt -f ${item.model.dockerfile} -t ${item.model.name}:latest ../../..
-      - docker run --rm -v $(realpath ${source.datasets_dir}):/datasets -v $(realpath ${source.models_dir}):/models -v $(realpath ${destination.output_dir}):/opt/ml/model ${item.model.name}:latest train --dataset-file ${item.dataset.container_path} --model-card-file ${item.model.container_path}
+    cmd:
+      - docker build -f $(dirname ${item.model.input_filename})/Dockerfile -t ${item.model.name}:latest $(dirname ${item.model.input_filename})
+      - docker run --rm -v $(realpath ${item.dataset.input_filename}):/$(basename ${item.dataset.input_filename}) -v $(realpath ${item.model.input_filename}):/$(basename ${item.model.input_filename}) -v $(realpath ${target.output_path}):/opt/ml/model ${item.model.name}:latest train --dataset-file /$(basename ${item.dataset.input_filename}) --model-card-file /$(basename ${item.model.input_filename})
       - docker image prune -a -f
-
     deps:
       - logs/setup.txt
-      - ${item.dataset.local_path}
-      - ${item.model.local_path}
-      - ${item.model.dockerfile}
+      - ${item.dataset.input_filename}
+      - ${item.model.input_filename}
     outs:
-      - ${destination.output_dir}/${item.dataset.name}_${item.model.name}.csv:
+      - ${target.output_path}/${item.dataset.name}_${item.model.name}.csv:
           cache: true
 
   calculate_metric:
     matrix: 
       dataset: ${datasets}
       model: ${models}
 
-    cmd: uv run proteingym-benchmark metric calc --output-path ${destination.output_dir}/${item.dataset.name}_${item.model.name}.csv --metric-path ${destination.metric_dir}/${item.dataset.name}_${item.model.name}.csv
+    cmd: uv run proteingym-benchmark metric calc --output-path ${target.output_path}/${item.dataset.name}_${item.model.name}.csv --metric-path ${target.metric_path}/${item.dataset.name}_${item.model.name}.csv
     deps:
-      - ${destination.output_dir}/${item.dataset.name}_${item.model.name}.csv
+      - ${target.output_path}/${item.dataset.name}_${item.model.name}.csv
     outs:
-      - ${destination.metric_dir}/${item.dataset.name}_${item.model.name}.csv:
-          cache: true
+      - ${target.metric_path}/${item.dataset.name}_${item.model.name}.csv:
+          cache: true

benchmark/zero_shot/local/default.yaml

@@ -0,0 +1,3 @@
+target:
+  output_path: output
+  metric_path: metric

benchmark/zero_shot/local/dvc.yaml

@@ -1,53 +1,42 @@
 vars:
-  - params.yaml
-
-  - datasets:
-    - name: ranganathan
-      container_path: /datasets/ranganathan/dataset.zip
-      local_path: ../../../datasets/ranganathan/dataset.zip
-
-  - models:
-    - name: esm
-      container_path: /models/esm/README.md
-      local_path: ../../../models/esm/README.md
-      dockerfile: ../../../models/esm/Dockerfile
+  - default.yaml
+  - datasets.json
+  - models.json
 
 stages:
 
   setup:
-    cmd: 
-      - mkdir -p logs ${destination.output_dir} ${destination.metric_dir}
+    cmd:
+      - mkdir -p logs ${target.output_path} ${target.metric_path}
       - echo "Created local directories" > logs/setup.txt
     outs:
       - logs/setup.txt
 
   create_training_job:
-    matrix: 
+    matrix:
       dataset: ${datasets}
       model: ${models}
-
-    cmd: 
-      - docker build --build-arg GIT_CACHE_BUST=${git.git_cache_bust} --secret id=git_auth,src=../git-auth.txt -f ${item.model.dockerfile} -t ${item.model.name}:latest ../../..
-      - docker run --rm -v $(realpath ${source.datasets_dir}):/datasets -v $(realpath ${source.models_dir}):/models -v $(realpath ${destination.output_dir}):/opt/ml/model ${item.model.name}:latest train --dataset-file ${item.dataset.container_path} --model-card-file ${item.model.container_path}
-      - docker image prune -a -f
 
+    cmd:
+      - docker build -f $(dirname ${item.model.input_filename})/Dockerfile -t ${item.model.name}:latest $(dirname ${item.model.input_filename})
+      - docker run --rm -v $(realpath ${item.dataset.input_filename}):/$(basename ${item.dataset.input_filename}) -v $(realpath ${item.model.input_filename}):/$(basename ${item.model.input_filename}) -v $(realpath ${target.output_path}):/opt/ml/model ${item.model.name}:latest train --dataset-file /$(basename ${item.dataset.input_filename}) --model-card-file /$(basename ${item.model.input_filename})
+      - docker image prune -a -f
     deps:
       - logs/setup.txt
-      - ${item.dataset.local_path}
-      - ${item.model.local_path}
-      - ${item.model.dockerfile}
+      - ${item.dataset.input_filename}
+      - ${item.model.input_filename}
     outs:
-      - ${destination.output_dir}/${item.dataset.name}_${item.model.name}.csv:
+      - ${target.output_path}/${item.dataset.name}_${item.model.name}.csv:
           cache: true
 
   calculate_metric:
     matrix: 
       dataset: ${datasets}
       model: ${models}
-    
-    cmd: uv run proteingym-benchmark metric calc --output-path ${destination.output_dir}/${item.dataset.name}_${item.model.name}.csv --metric-path ${destination.metric_dir}/${item.dataset.name}_${item.model.name}.csv
+
+    cmd: uv run proteingym-benchmark metric calc --output-path ${target.output_path}/${item.dataset.name}_${item.model.name}.csv --metric-path ${target.metric_path}/${item.dataset.name}_${item.model.name}.csv
     deps:
-      - ${destination.output_dir}/${item.dataset.name}_${item.model.name}.csv
+      - ${target.output_path}/${item.dataset.name}_${item.model.name}.csv
     outs:
-      - ${destination.metric_dir}/${item.dataset.name}_${item.model.name}.csv:
-          cache: true
+      - ${target.metric_path}/${item.dataset.name}_${item.model.name}.csv:
+          cache: true

dist/.gitignore

@@ -0,0 +1 @@
+*

models/esm/Dockerfile

@@ -11,8 +11,7 @@ COPY dist/proteingym*.whl dist/
 # Copy and install dependencies
 COPY README.md README.md
 COPY pyproject.toml pyproject.toml
-RUN uv sync --no-cache
-
 COPY src/ src/
+RUN uv sync --no-cache
 
 ENTRYPOINT ["uv", "run", "esm"]

models/esm/dist/.gitignore

@@ -0,0 +1 @@
+*