Merge pull request #638 from instructlab/constraints-dev

Author: bbrowning

.github/workflows/constraints-update.yml

@@ -0,0 +1,32 @@
+name: Update constraints-dev.txt
+
+on:
+  schedule:
+    - cron: '0 3 * * 1'  # Every Monday at 03:00 UTC
+  workflow_dispatch:
+
+jobs:
+  update-constraints:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - name: Checkout "update-constraints" in-house CI action
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          repository: instructlab/ci-actions
+          path: ci-actions
+          ref: v0.2.0
+          sparse-checkout: |
+            actions/update-constraints
+
+      - name: Update constraints
+        id: update-constraints
+        uses: ./ci-actions/actions/update-constraints
+        with:
+          gh-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}

.github/workflows/e2e-nvidia-l4-x1.yml

@@ -105,7 +105,7 @@ jobs:
           path: "instructlab"
           # https://github.com/actions/checkout/issues/249
           fetch-depth: 0
-  
+
       - name: Checkout instructlab/sdg
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
@@ -125,26 +125,27 @@ jobs:
       - name: Install ilab
         working-directory: ./instructlab
         run: |
-          export CUDA_HOME="/usr/local/cuda"
-          export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64"
-          export PATH="$PATH:$CUDA_HOME/bin"
-          python3.11 -m venv --upgrade-deps venv
-          . venv/bin/activate
-          nvidia-smi
-          python3.11 -m pip cache remove llama_cpp_python
+          PYTHON=python3.11 ./scripts/install-ilab-with-cuda.sh
 
-          CMAKE_ARGS="-DGGML_CUDA=on" python3.11 -m pip install -v . -c constraints-dev.txt
-
-          # https://github.com/instructlab/instructlab/issues/1821
-          # install with Torch and build dependencies installed
-          python3.11 -m pip install -v packaging wheel setuptools-scm
-          python3.11 -m pip install -v .[cuda] -r requirements-vllm-cuda.txt -c constraints-dev.txt
-  
       - name: Update instructlab-sdg library
         working-directory: ./sdg
         run: |
           . ../instructlab/venv/bin/activate
-          pip install  -v .
+          # Patch out our own pin from the ilab repo constraints file
+          ilab_constraints=../instructlab/constraints-dev.txt
+          sed -i '/instructlab-sdg==/d' $ilab_constraints
+          # Since we reuse the virtual environment prepared using ilab
+          # constraints, we should stick to the same constraints when
+          # installing latest sdg.
+          #
+          # FIX: this is not ideal; a proper fix would require decoupling the
+          # two repos in CI: either by removing the job completely and relying
+          # on "sdk" (no ilab) test runs; or by preparing a separate
+          # constraints file that would consider both the requirements files
+          # for the sdg library AND for the ilab - so that they are
+          # consistent.
+          pip_install="pip install -c $ilab_constraints"
+          $pip_install .
 
       - name: Check disk before tests
         run: |

.github/workflows/e2e-nvidia-l40s-x4.yml

@@ -101,7 +101,7 @@ jobs:
           path: "instructlab"
           # https://github.com/actions/checkout/issues/249
           fetch-depth: 0
-  
+
       - name: Checkout instructlab/sdg
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
@@ -180,8 +180,21 @@ jobs:
         working-directory: ./sdg
         run: |
           . ../instructlab/venv/bin/activate
-          pip install .
-          pip install .[cuda]
+          # Patch out our own pin from the ilab repo constraints file
+          ilab_constraints=../instructlab/constraints-dev.txt
+          sed -i '/instructlab-sdg==/d' $ilab_constraints
+          # Since we reuse the virtual environment prepared using ilab
+          # constraints, we should stick to the same constraints when
+          # installing latest sdg.
+          #
+          # FIX: this is not ideal; a proper fix would require decoupling the
+          # two repos in CI: either by removing the job completely and relying
+          # on "sdk" (no ilab) test runs; or by preparing a separate
+          # constraints file that would consider both the requirements files
+          # for the sdg library AND for the ilab - so that they are
+          # consistent.
+          pip_install="pip install -c $ilab_constraints"
+          $pip_install .
 
       - name: Check disk before tests
         run: |

.github/workflows/e2e-nvidia-t4-x1.yml

@@ -112,24 +112,27 @@ jobs:
       - name: Install ilab
         working-directory: ./instructlab
         run: |
-          export PATH="/home/ec2-user/.local/bin:/usr/local/cuda/bin:$PATH"
-          python3.11 -m venv --upgrade-deps venv
-          . venv/bin/activate
-          nvidia-smi
-          python3.11 -m pip cache remove llama_cpp_python
-
-          CMAKE_ARGS="-DGGML_CUDA=on" python3.11 -m pip install -v . -c constraints-dev.txt
+          PYTHON=python3.11 ./scripts/install-ilab-with-cuda.sh
 
-          # https://github.com/instructlab/instructlab/issues/1821
-          # install with Torch and build dependencies installed
-          python3.11 -m pip install -v packaging wheel setuptools-scm
-          python3.11 -m pip install -v .[cuda]
-        
       - name: Update instructlab-sdg library
         working-directory: ./sdg
         run: |
           . ../instructlab/venv/bin/activate
-          pip install .
+          # Patch out our own pin from the ilab repo constraints file
+          ilab_constraints=../instructlab/constraints-dev.txt
+          sed -i '/instructlab-sdg==/d' $ilab_constraints
+          # Since we reuse the virtual environment prepared using ilab
+          # constraints, we should stick to the same constraints when
+          # installing latest sdg.
+          #
+          # FIX: this is not ideal; a proper fix would require decoupling the
+          # two repos in CI: either by removing the job completely and relying
+          # on "sdk" (no ilab) test runs; or by preparing a separate
+          # constraints file that would consider both the requirements files
+          # for the sdg library AND for the ilab - so that they are
+          # consistent.
+          pip_install="pip install -c $ilab_constraints"
+          $pip_install .
 
       - name: Check disk before tests
         run: |

.github/workflows/functional-gpu-nvidia-t4-x1.yml

@@ -18,7 +18,8 @@ on:
       - "**.py"
       - "pyproject.toml"
       - "requirements**.txt"
-      - 'tox.ini'
+      - "constraints-dev.txt"
+      - "tox.ini"
       - ".github/workflows/functional-gpu-nvidia-t4-x1.yml" # This workflow
 
 concurrency:
@@ -103,10 +104,10 @@ jobs:
           python3.11 -m venv --upgrade-deps venv
           . venv/bin/activate
           nvidia-smi
-          python3.11 -m pip install tox tox-gh>=1.2
+          python3.11 -m pip install tox tox-gh>=1.2 -c constraints-dev.txt
           python3.11 -m pip cache remove llama_cpp_python
 
-          CMAKE_ARGS="-DLLAMA_CUDA=on" python3.11 -m pip install -r requirements-dev.txt
+          CMAKE_ARGS="-DLLAMA_CUDA=on" python3.11 -m pip install -r requirements-dev.txt -c constraints-dev.txt
 
       - name: Check disk before tests
         run: |

.github/workflows/lint.yml

@@ -11,6 +11,7 @@ on:
       - '**.py'
       - 'pyproject.toml'
       - 'requirements*.txt'
+      - 'constraints-dev.txt'
       - 'tox.ini'
       - '.pylintrc'
       - 'scripts/*.sh'
@@ -24,6 +25,7 @@ on:
       - '**.py'
       - 'pyproject.toml'
       - 'requirements*.txt'
+      - 'constraints-dev.txt'
       - 'tox.ini'
       - '.pylintrc'
       - 'scripts/*.sh'
@@ -71,7 +73,7 @@ jobs:
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           # https://github.com/actions/checkout/issues/249
-          fetch-depth: 0  
+          fetch-depth: 0
 
       - name: Setup Python 3.11
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
@@ -85,7 +87,7 @@ jobs:
       - name: "Install tox"
         run: |
           python -m pip install --upgrade pip
-          python -m pip install tox tox-gh
+          python -m pip install tox tox-gh -c constraints-dev.txt
 
       - name: "${{ matrix.lint.name }}"
         run: |

.github/workflows/test.yml

@@ -12,6 +12,7 @@ on:
       - '**.py'
       - 'pyproject.toml'
       - 'requirements*.txt'
+      - 'constraints-dev.txt'
       - 'tox.ini'
       - '.github/workflows/test.yml' # This workflow
   pull_request:
@@ -22,6 +23,7 @@ on:
       - '**.py'
       - 'pyproject.toml'
       - 'requirements*.txt'
+      - 'constraints-dev.txt'
       - 'tox.ini'
       - '.github/workflows/test.yml' # This workflow
 
@@ -101,7 +103,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          python -m pip install tox tox-gh>=1.2
+          python -m pip install tox tox-gh>=1.2 -c constraints-dev.txt
 
       - name: Run unit and functional tests with tox
         run: |

.gitignore

@@ -169,4 +169,10 @@ cython_debug/
 .vscode/
 
 # SDG examples output
-docs/examples/**/output
+docs/examples/**/output
+
+# Mac stuff
+**/.DS_Store
+
+# pyenv version
+.python-version