bionemo-framework/.github/workflows/convergence-tests.yml at 3d0368057ef73ab8070a3969926f0b971769e0c6 · NVIDIA/bionemo-framework · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
name: "BioNeMo Model Convergence Tests"

on:
  workflow_dispatch:
    inputs:
      branch:
        description: "Branch to use (ignored if commit SHA is provided)"
        required: true
        default: "main"
        type: string
      commit_sha:
        description: "Commit SHA (optional - overrides branch if provided)"
        required: false
        type: string
      model_config:
        description: "Model configuration to use"
        required: true
        default: "esm2_accelerate"
        type: choice
        options:
          - amplify_accelerate_te_fp8.yaml
          - esm2_accelerate
          - esm2_native_te_mfsdp
          - esm2_native_te_nvfsdp_thd
          - geneformer_native_te_nvfsdp_fp8
      config_override:
        description: "Custom config override (optional, e.g. 'geneformer_106m')"
        required: false
        type: string

jobs:
  submit-lepton-jobs:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"
          cache: "pip"
          cache-dependency-path: |
            requirements.ci.txt
            ci/lepton/model_convergence/scripts/requirements.txt

      - name: Install deps
        run: |
          python -m pip install --upgrade pip
          pip install "hydra-core==1.3.2" "omegaconf==2.3.0" "leptonai" typer pandas
          python - <<'PY'
          import hydra, omegaconf, leptonai
          print("OK:", hydra.__version__, omegaconf.__version__, leptonai.__version__)
          PY

      - name: Submit Lepton Jobs
        env:
          LEP_LOGIN_CREDENTIALS: ${{ secrets.LEP_LOGIN_CREDENTIALS }}
        run: |
          lep login -c "$LEP_LOGIN_CREDENTIALS" || true
          if [ -n "${{ inputs.config_override }}" ]; then
            python ci/lepton/model_convergence/scripts/launch_job.py \
              --config-name="${{ inputs.config_override }}.yaml" \
              branch="${{ inputs.branch }}" \
              commit_sha="${{ inputs.commit_sha }}"
          else
            python ci/lepton/model_convergence/scripts/launch_job.py \
              --config-name="recipes/${{ inputs.model_config }}" \
              branch="${{ inputs.branch }}" \
              commit_sha="${{ inputs.commit_sha }}"
          fi