Merge branch 'main' into dev/ba/BIONEMO-2473-add-evo2-lora-tests

Author: balvisio

.github/labels.yml

@@ -0,0 +1,20 @@
+# Test pipeline labels
+- name: ciflow:slow
+  description: Run slow single GPU integration tests marked as @pytest.mark.slow for bionemo2
+  color: 0E8A16  # Green
+
+- name: ciflow:notebooks
+  description: Run Jupyter notebooks execution tests for bionemo2
+  color: 1D76DB  # Blue
+
+- name: ciflow:all
+  description: Run all tests (unit tests, slow tests, and notebooks) for bionemo2. This label can be used to enforce running tests for all bionemo2.
+  color: FBCA04  # Yellow/Gold
+
+- name: ciflow:all-recipes
+  description: Run tests for all recipes (under bionemo-recipes). This label can be used to enforce running tests for all recipes.
+  color: 812afe  # Purple
+
+- name: ciflow:skip
+  description: Skip all CI tests for this PR
+  color: B60205  # Red

.github/pull_request_template.md

@@ -2,6 +2,14 @@
 
 <!-- Provide a detailed description of the changes in this PR -->
 
+#### Usage
+
+<!--- How does a user interact with the changed code -->
+
+```python
+TODO: Add code snippet
+```
+
 ### Type of changes
 
 <!-- Mark the relevant option with an [x] -->
@@ -14,14 +22,20 @@
 
 ### CI Pipeline Configuration
 
-Configure CI behavior by applying the relevant labels:
+Configure CI behavior by applying the relevant labels. By default, only basic unit tests are run.
+
+- [ciflow:skip](https://github.com/NVIDIA/bionemo-framework/blob/main/docs/docs/main/contributing/contributing.md#ciflow:skip) - Skip all CI tests for this PR
+- [ciflow:notebooks](https://github.com/NVIDIA/bionemo-framework/blob/main/docs/docs/main/contributing/contributing.md#ciflow:notebooks) - Run Jupyter notebooks execution tests for bionemo2
+- [ciflow:slow](https://github.com/NVIDIA/bionemo-framework/blob/main/docs/docs/main/contributing/contributing.md#ciflow:slow) - Run slow single GPU integration tests marked as @pytest.mark.slow for bionemo2
+- [ciflow:all](https://github.com/NVIDIA/bionemo-framework/blob/main/docs/docs/main/contributing/contributing.md#ciflow:all) - Run all tests (unit tests, slow tests, and notebooks) for bionemo2. This label can be used to enforce running tests for all bionemo2.
+- [ciflow:all-recipes](https://github.com/NVIDIA/bionemo-framework/blob/main/docs/docs/main/contributing/contributing.md#ciflow:all-recipes) - Run tests for all recipes (under bionemo-recipes). This label can be used to enforce running tests for all recipes.
 
-- [SKIP_CI](https://github.com/NVIDIA/bionemo-framework/blob/main/docs/docs/user-guide/contributing/contributing.md#skip_ci) - Skip all continuous integration tests
-- [INCLUDE_NOTEBOOKS_TESTS](https://github.com/NVIDIA/bionemo-framework/blob/main/docs/docs/user-guide/contributing/contributing.md#include_notebooks_tests) - Execute notebook validation tests in pytest
-- [INCLUDE_SLOW_TESTS](https://github.com/NVIDIA/bionemo-framework/blob/main/docs/docs/user-guide/contributing/contributing.md#include_slow_tests) - Execute tests labelled as slow in pytest for extensive testing
+Unit tests marked as `@pytest.mark.multi_gpu` or `@pytest.mark.distributed` are not run in the PR pipeline.
+
+For more details, see [CONTRIBUTING](CONTRIBUTING.md)
 
 > [!NOTE]
-> By default, the notebooks validation tests are skipped unless explicitly enabled.
+> By default, only basic unit tests are run. Add appropriate labels to enable an additional test coverage.
 
 #### Authorizing CI Runs
 
@@ -33,14 +47,6 @@ runs on NVIDIA's compute resources.
 - If a pull request is opened by an untrusted user or contains untrusted changes, an NVIDIA org member must leave an
   `/ok to test` comment on the pull request to trigger CI. This will need to be done for each new commit.
 
-### Usage
-
-<!--- How does a user interact with the changed code -->
-
-```python
-# TODO: Add code snippet
-```
-
 ### Pre-submit Checklist
 
 <!--- Ensure all items are completed before submitting -->

.github/workflows/approvals.yml

@@ -4,7 +4,7 @@ on:
   pull_request_review:
 
 env:
-  TIER2_REVIEWERS: "jstjohn,trvachov,pstjohn"
+  TIER2_REVIEWERS: "jstjohn,trvachov,pstjohn,dorotat-nv"
 
 jobs:
   check_approval:

.github/workflows/sync-labels.yml

@@ -0,0 +1,26 @@
+name: Sync Labels
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - '.github/labels.yml'
+      - '.github/workflows/sync-labels.yml'
+  workflow_dispatch:
+
+jobs:
+  sync-labels:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Sync labels
+        uses: micnncim/action-label-syncer@v1
+        with:
+          repository: ${{ github.repository }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+          config_file: .github/labels.yml
+          # Optionally you can add these flags:
+          # prune: true   # Delete labels not defined in your config file
+          # dry_run: true # Preview changes without applying them

.github/workflows/unit-tests-framework.yml

@@ -1,3 +1,35 @@
+# BioNeMo Framework CI Workflow
+#
+# This workflow runs comprehensive tests for the BioNeMo framework on various triggers:
+#
+# TRIGGERS:
+# - Push to main branch, pull-request branches, or dependabot branches
+# - Merge group events (when PRs are merged via merge queue)
+# - Scheduled runs (daily at 7 AM UTC)
+#
+# WORKFLOW OVERVIEW:
+# 1. changed-files: Detects which files have changed compared to main branch
+#    - Tracks changes in: All files (**) except:
+#      * bionemo-recipes/** (recipes have separate workflow)
+#      * **.md (markdown documentation files)
+#      * .github/** (GitHub configuration, except this workflow file)
+#      * .gitignore, .devcontainer/**
+#      * ci/scripts/recipes_local_test.py
+#    - Includes: .github/workflows/unit-tests-framework.yml (this workflow file)
+# 2. pre-commit: Runs static code checks and linting
+# 3. get-pr-labels: Retrieves PR labels for conditional job execution
+# 4. build-bionemo-image: Builds Docker image (conditional on triggers/labels)
+# 5. run-tests: Runs unit tests (when image build succeeds)
+# 6. run-tests-slow: Runs slow tests (when image succeeds + ciflow:slow label OR schedule/merge_group/ciflow:all)
+# 7. run-tests-notebooks: Runs notebook tests (when image succeeds + ciflow:notebooks label OR schedule/merge_group/ciflow:all)
+# 8. verify-tests-status: Verifies all test jobs completed successfully
+#
+# CONDITIONAL EXECUTION:
+# - build-bionemo-image runs on: schedule, ciflow:all label, (no ciflow:skip + modified files), (merge_group + modified files)
+# - run-tests runs when: build-bionemo-image succeeds
+# - run-tests-slow runs when: build-bionemo-image succeeds AND (schedule OR merge_group OR ciflow:all OR ciflow:slow)
+# - run-tests-notebooks runs when: build-bionemo-image succeeds AND (schedule OR merge_group OR ciflow:all OR ciflow:notebooks)
+
 name: "BioNeMo Framework CI"
 
 on:
@@ -82,7 +114,7 @@ jobs:
   get-pr-labels:
     runs-on: ubuntu-latest
     outputs:
-      labels: ${{ steps.get-labels.outputs.labels }}
+      labels: ${{ steps.get-labels.outputs.labels || steps.get-labels-empty.outputs.labels }}
     steps:
       - name: Get PR number from branch
         if: startsWith(github.ref, 'refs/heads/pull-request/')
@@ -99,21 +131,34 @@ jobs:
         run: |
           LABELS=$(gh api repos/${{ github.repository }}/pulls/${{ steps.get-pr-num.outputs.pr_num }} --jq '[.labels[].name]' || echo "[]")
           echo "labels=$LABELS" >> $GITHUB_OUTPUT
+          echo "Retrieved labels: $LABELS"
 
       - name: Set empty labels for non-PR branches
         if: ${{ !startsWith(github.ref, 'refs/heads/pull-request/') }}
         id: get-labels-empty
-        run: echo "labels=[]" >> $GITHUB_OUTPUT
+        run: |
+          echo "labels=[]" >> $GITHUB_OUTPUT
+          echo "Set empty labels for non-PR branch"
 
   build-bionemo-image:
     needs:
       - pre-commit
       - get-pr-labels
       - changed-files
     runs-on: linux-amd64-cpu16
-    # We want this to run on schedule events even if no files have changed
-    if: ${{ github.event_name == 'schedule' || (!contains(fromJSON(needs.get-pr-labels.outputs.labels || '[]'), 'SKIP_CI') && needs.changed-files.outputs.any_changed == 'true') }}
+    if: |
+      (github.event_name == 'schedule') ||
+      contains(fromJSON(needs.get-pr-labels.outputs.labels || '[]'), 'ciflow:all') ||
+      (
+        !contains(fromJSON(needs.get-pr-labels.outputs.labels || '[]'), 'ciflow:skip') &&
+        (needs.changed-files.outputs.any_changed == 'true')
+      ) ||
+      (
+        (github.event_name == 'merge_group') &&
+        (needs.changed-files.outputs.any_changed == 'true')
+      )
     steps:
+
       - name: Login to Docker Hub
         uses: docker/login-action@v3
         with:
@@ -171,6 +216,7 @@ jobs:
       credentials:
         username: ${{ vars.DOCKER_USERNAME }}
         password: ${{ secrets.DOCKER_PASSWORD }}
+    if: needs.build-bionemo-image.result == 'success'
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -200,7 +246,7 @@ jobs:
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
 
-  run-slow-tests:
+  run-tests-slow:
     needs:
       - build-bionemo-image
       - get-pr-labels
@@ -211,13 +257,19 @@ jobs:
         username: ${{ vars.DOCKER_USERNAME }}
         password: ${{ secrets.DOCKER_PASSWORD }}
     if: |
-      github.event_name == 'schedule' || github.event_name == 'merge_group' ||
-      contains(fromJSON(needs.get-pr-labels.outputs.labels || '[]'), 'INCLUDE_SLOW_TESTS')
+        (needs.build-bionemo-image.result == 'success') &&
+        (
+          (github.event_name == 'schedule') ||
+          (github.event_name == 'merge_group') ||
+          contains(fromJSON(needs.get-pr-labels.outputs.labels || '[]'), 'ciflow:all') ||
+          contains(fromJSON(needs.get-pr-labels.outputs.labels || '[]'), 'ciflow:slow')
+        )
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
 
       - name: Run slow tests
+
         env:
           BIONEMO_DATA_SOURCE: ngc
         # Not every sub-package has slow tests, and since some sub-packages have tests under the same name we need
@@ -227,14 +279,20 @@ jobs:
           ./ci/scripts/run_pytest_slow.sh
 
 
-  run-notebooks-docs:
+  run-tests-notebooks:
     needs:
       - build-bionemo-image
       - get-pr-labels
     runs-on: linux-amd64-gpu-l4-latest-1
+
     if: |
-      github.event_name == 'schedule' || github.event_name == 'merge_group' ||
-      contains(fromJSON(needs.get-pr-labels.outputs.labels || '[]'), 'INCLUDE_NOTEBOOKS_TESTS')
+        (needs.build-bionemo-image.result == 'success') &&
+        (
+          (github.event_name == 'schedule') ||
+          (github.event_name == 'merge_group') ||
+          contains(fromJSON(needs.get-pr-labels.outputs.labels || '[]'), 'ciflow:all') ||
+          contains(fromJSON(needs.get-pr-labels.outputs.labels || '[]'), 'ciflow:notebooks')
+        )
     container:
       image: svcbionemo023/bionemo-framework:${{ github.run_id }}
       credentials:
@@ -260,8 +318,8 @@ jobs:
       - get-pr-labels
       - build-bionemo-image
       - run-tests
-      - run-slow-tests
-      - run-notebooks-docs
+      - run-tests-slow
+      - run-tests-notebooks
       # Add all other run-*-test jobs
     runs-on: ubuntu-latest
     if: always() # This ensures the job runs even if previous jobs fail

.github/workflows/unit-tests-recipes.yml

@@ -70,12 +70,12 @@ jobs:
           if [[ "$EVENT_NAME" == "schedule" ]]; then
             DIRS=$(echo "$ALL_DIRS")
           else
-            # Check if INCLUDE_ALL_RECIPES label is present
+            # Check if "ciflow:all-recipes" label is present
             HAS_INCLUDE_ALL_LABEL=false
             if [[ "$PR_INFO" != "null" && "$PR_INFO" != "" ]]; then
-              if echo "$PR_INFO" | jq -e '.labels[]? | select(.name == "INCLUDE_ALL_RECIPES")' > /dev/null 2>&1; then
+              if echo "$PR_INFO" | jq -e '.labels[]? | select(.name == "ciflow:all-recipes")' > /dev/null 2>&1; then
                 HAS_INCLUDE_ALL_LABEL=true
-                echo "Found INCLUDE_ALL_RECIPES label - running all directories"
+                echo "Found 'ciflow:all-recipes' label - running all directories"
               fi
             fi
 

docs/docs/main/about/background/SUMMARY.md

@@ -1,2 +1,2 @@
-- [NeMo2 Parallelism](nemo2.md)
+- [NeMo2](nemo2.md)
 - [Megatron Dataset Considerations](megatron_datasets.md)