[Deps] Split up of dependencies

.github/Dockerfile.ci

@@ -44,3 +44,9 @@ COPY --from=ci-build $TTFORGE_TOOLCHAIN_DIR $TTFORGE_TOOLCHAIN_DIR
 
 RUN du -h --max-depth=2 $TTMLIR_TOOLCHAIN_DIR
 RUN du -h --max-depth=2 $TTFORGE_TOOLCHAIN_DIR
+
+# Install all dev/test requirements (fat image)
+COPY env/dev_requirements.txt /tmp/dev_requirements.txt
+COPY env/core_requirements.txt /tmp/core_requirements.txt
+RUN pip install --ignore-installed -r /tmp/dev_requirements.txt
+RUN pip install --ignore-installed -r /tmp/core_requirements.txt

.github/Dockerfile.ird

@@ -1,4 +1,4 @@
-ARG FROM_IMAGE=base
+ARG FROM_IMAGE=ci
 ARG FROM_TAG=latest
 
 FROM ghcr.io/tenstorrent/tt-forge-fe/tt-forge-fe-${FROM_IMAGE}-ubuntu-22-04:${FROM_TAG}

.github/Dockerfile.slimci

@@ -0,0 +1,12 @@
+FROM ghcr.io/tenstorrent/tt-forge-fe/tt-forge-fe-base-ubuntu-22-04:latest AS slimci
+SHELL ["/bin/bash", "-c"]
+
+# Set environment variables
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Copy requirements
+COPY env/core_requirements.txt /tmp/core_requirements.txt
+
+# Install python packages for compiler/core only
+RUN pip install --upgrade pip setuptools wheel && \
+    pip install --ignore-installed -r /tmp/core_requirements.txt

.github/build-docker-images.sh

@@ -10,6 +10,7 @@ BASE_IMAGE_NAME=ghcr.io/$REPO/tt-forge-fe-base-ubuntu-22-04
 CI_IMAGE_NAME=ghcr.io/$REPO/tt-forge-fe-ci-ubuntu-22-04
 BASE_IRD_IMAGE_NAME=ghcr.io/$REPO/tt-forge-fe-base-ird-ubuntu-22-04
 IRD_IMAGE_NAME=ghcr.io/$REPO/tt-forge-fe-ird-ubuntu-22-04
+SLIM_IMAGE_NAME=ghcr.io/$REPO/tt-forge-fe-slimci-ubuntu-22-04
 
 # Compute the hash of the Dockerfile
 DOCKER_TAG=$(./.github/get-docker-tag.sh)
@@ -40,6 +41,7 @@ build_and_push $BASE_IMAGE_NAME .github/Dockerfile.base
 build_and_push $BASE_IRD_IMAGE_NAME .github/Dockerfile.ird base
 build_and_push $CI_IMAGE_NAME .github/Dockerfile.ci
 build_and_push $IRD_IMAGE_NAME .github/Dockerfile.ird ci
+build_and_push $SLIM_IMAGE_NAME .github/Dockerfile.slimci
 
 echo "All images built and pushed successfully"
 echo "CI_IMAGE_NAME:"

.github/workflows/fail-inspector.yml

@@ -83,8 +83,9 @@ jobs:
           shell: bash
           run: |
             source env/activate
-            pip install tt_tvm*.whl --force-reinstall
-            pip install tt_forge_fe*.whl --force-reinstall
+            pip install ./tt_tvm*.whl --force-reinstall
+            pip install ./tt_forge_fe*.whl --force-reinstall
+            pip install -r env/dev_requirements.txt
 
         - name: Run Test
           env:

.github/workflows/model-analysis.yml

@@ -171,8 +171,9 @@ jobs:
         shell: bash
         run: |
           source env/activate
-          pip install tt_tvm*.whl --force-reinstall
-          pip install tt_forge_fe*.whl --force-reinstall
+          pip install ./tt_tvm*.whl --force-reinstall
+          pip install ./tt_forge_fe*.whl --force-reinstall
+          pip install -r env/dev_requirements.txt
 
       - name: Generate Models Ops tests
         shell: bash

.github/workflows/perf-benchmark-sub.yml

@@ -133,8 +133,9 @@ jobs:
       shell: bash
       run: |
         source env/activate
-        pip install tt_tvm*.whl --force-reinstall
-        pip install tt_forge*.whl --force-reinstall
+        pip install ./tt_tvm*.whl --force-reinstall
+        pip install ./tt_forge*.whl --force-reinstall
+        pip install -r env/dev_requirements.txt
 
     - name: Run Perf Benchmark
       env:

.github/workflows/test-model-analysis-sub.yml

@@ -140,8 +140,9 @@ jobs:
         shell: bash
         run: |
           source env/activate
-          pip install tt_tvm*.whl --force-reinstall
-          pip install tt_forge_fe*.whl --force-reinstall
+          pip install ./tt_tvm*.whl --force-reinstall
+          pip install ./tt_forge_fe*.whl --force-reinstall
+          pip install -r env/dev_requirements.txt
 
 
       - name: Extract and Export Unique Ops Configuration

.github/workflows/test-sub.yml

@@ -20,7 +20,8 @@
         type: string
       docker-image:
         description: 'Docker image to use for build'
-        required: true
+        required: false
+        default: 'ghcr.io/tenstorrent/tt-forge-fe/tt-forge-fe-ci-ubuntu-22-04:latest'
         type: string
       runs-on:
         description: 'Runs on'
@@ -67,7 +68,7 @@
     continue-on-error: ${{ inputs.allow-fail }}
 
     container:
-      image: ${{ inputs.sh-runner && format('harbor.ci.tenstorrent.net/{0}', inputs.docker-image) || inputs.docker-image }}
+      image: ${{ (inputs.test_mark == 'slim_wheel') && 'ghcr.io/tenstorrent/tt-forge-fe/tt-forge-fe-slimci-ubuntu-22-04:latest' || inputs.docker-image }}
       options: --device /dev/tenstorrent/0
       volumes:
         - /dev/hugepages:/dev/hugepages
@@ -122,6 +123,22 @@
           git submodule foreach --recursive git clean -ffdx
           git submodule foreach --recursive git reset --hard
 
+      - name: Create temporary directory for artifacts
+        run: mkdir -p ./artifacts/
+
+      - name: Debug - Check inputs and download strategy
+        shell: bash
+        run: |
+          echo "=== Download Strategy Debug ==="
+          echo "inputs.run_id: '${{ inputs.run_id }}'"
+          if [ -n "${{ inputs.run_id }}" ]; then
+            echo "Will use tenstorrent/tt-forge/.github/actions/download-artifact@main"
+            echo "Looking for run_id: ${{ inputs.run_id }}"
+          else
+            echo "Will use dawidd6/action-download-artifact@v9"
+            echo "Looking for workflow: on-push.yml in repo: tenstorrent/tt-forge-fe"
+          fi
+
       - name: Download wheel
         if: ${{ inputs.run_id }}
         continue-on-error: true
@@ -130,6 +147,7 @@
           name: forge-wheel
           run_id: ${{ inputs.run_id }}
           github_token: ${{ secrets.GITHUB_TOKEN }}
+          path: ./artifacts/
 
       - name: Find and download forge wheel
         if: ${{ !inputs.run_id }}
@@ -143,15 +161,130 @@
           repo: tenstorrent/tt-forge-fe
           check_artifacts: true
           search_artifacts: true
+          path: ./artifacts/
+
+      - name: Debug - Check downloaded artifacts
+        shell: bash
+        run: |
+          echo "=== Checking artifacts directory ==="
+          if [ -d "./artifacts/" ]; then
+            echo "Artifacts directory exists"
+            ls -la "./artifacts/"
+            echo ""
+            echo "=== Looking for wheel files ==="
+            find "./artifacts/" -name "*.whl" -type f | while read wheel; do
+              echo "Found wheel: $wheel"
+              echo "  Size: $(stat -c%s "$wheel") bytes"
+              echo "  Full path: $(realpath "$wheel")"
+            done
+          else
+            echo "ERROR: Artifacts directory does not exist!"
+          fi
+          echo ""
+          echo "=== Current working directory ==="
+          pwd
+          echo ""
+          echo "=== Contents of current directory ==="
+          ls -la .
 
       - name: Install wheel
         shell: bash
         run: |
           source env/activate
-          pip install tt_tvm*.whl --upgrade
-          pip install tt_forge_fe*.whl --upgrade
+          # Install tt_tvm wheel if it exists
+          TT_TVM_WHEEL=$(find "./artifacts/" -name "tt_tvm*.whl" | head -1)
+          if [ -f "$TT_TVM_WHEEL" ]; then
+            echo "Installing tt_tvm wheel: $TT_TVM_WHEEL"
+            pip install "$TT_TVM_WHEEL" --upgrade
+          else
+            echo "No tt_tvm wheel found in ./artifacts/"
+            ls -la "./artifacts/" || echo "Artifacts directory does not exist"
+          fi
+
+          # Install tt_forge_fe wheel if it exists
+          TT_FORGE_FE_WHEEL=$(find "./artifacts/" -name "tt_forge_fe*.whl" | head -1)
+          if [ -f "$TT_FORGE_FE_WHEEL" ]; then
+            echo "Installing tt_forge_fe wheel: $TT_FORGE_FE_WHEEL"
+            pip install "$TT_FORGE_FE_WHEEL" --upgrade
+          else
+            echo "No tt_forge_fe wheel found in ./artifacts/"
+            ls -la "./artifacts/" || echo "Artifacts directory does not exist"
+          fi
+
+      - name: Run Sanity Tests
+        if: ${{ inputs.test_mark == 'slim_wheel' }}
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          HF_HOME: /mnt/dockercache/huggingface
+          FORGE_MODELS_CACHE: /mnt/dockercache/forge_models_cache
+          HF_HUB_DISABLE_PROGRESS_BARS: 1
+          FORGE_DISABLE_REPORTIFY_DUMP: 1
+        shell: bash
+        run: |
+          source env/activate
+          echo "Collecting slim wheel tests..."
+          pytest --splits 1 \
+               --group 1 \
+               --splitting-algorithm least_duration \
+               -m "slim_wheel" --collect-only -q forge/test/test_slim_wheel.py \
+                | sed -n '/^Collected tests /,/^collected /p' | sed '/^[Cc]ollected /d' >.pytest_slim_wheel_tests_to_run
+
+          echo "Collected slim wheel tests."
+          cat .pytest_slim_wheel_tests_to_run
+
+          set +e
+          python .github/workflows/test_runner.py  \
+               --junit-xml=${{ steps.strings.outputs.test_report_path }}_slim_wheel \
+               --log-memory-usage \
+               $(if [ "${{inputs.continue-on-crash}}" = "true" ]; then echo "--continue-after-crash"; fi)
+          slim_wheel_exit_code=$?
+
+          if [ -f "crashed_pytest.log" ]; then
+            echo "Creating crash summary for slim wheel tests..."
+            echo "### Crashed slim wheel tests:" >>$GITHUB_STEP_SUMMARY
+            cat crashed_pytest.log >>$GITHUB_STEP_SUMMARY
+            mv crashed_pytest.log crashed_pytest_slim_wheel.log
+          fi
+
+          if [ $slim_wheel_exit_code -ne 0 ]; then
+            echo "Slim wheel tests failed with exit code $slim_wheel_exit_code"
+            exit $slim_wheel_exit_code
+          fi
+
+      - name: Upload Slim Wheel Test Log
+        uses: actions/upload-artifact@v4
+        if: success() || failure()
+        with:
+          name: slim_wheel-test-log-${{ matrix.build.runs-on }}-${{ matrix.test_group_id }}-${{ inputs.test_mark }}-${{ steps.fetch-job-id.outputs.job_id }}
+          path: pytest.log
+
+      - name: Upload Slim Wheel Test Crash Log
+        uses: actions/upload-artifact@v4
+        if: success() || failure()
+        with:
+          name: slim_wheel-test-crash-log-${{ matrix.build.runs-on }}-${{ matrix.test_group_id }}-${{ inputs.test_mark }}-${{ steps.fetch-job-id.outputs.job_id }}
+          path: crashed_pytest_slim_wheel.log
+
+      - name: Upload Slim Wheel Test Report
+        uses: actions/upload-artifact@v4
+        if: success() || failure()
+        with:
+          name: slim_wheel-test-reports-${{ matrix.build.runs-on }}-${{ matrix.test_group_id }}-${{ steps.fetch-job-id.outputs.job_id }}
+          path: ${{ steps.strings.outputs.test_report_path }}_slim_wheel
+
+      - name: Show Slim Wheel Test Report
+        uses: mikepenz/action-junit-report@v5
+        if: success() || failure()
+        with:
+          report_paths: ${{ steps.strings.outputs.test_report_path }}_slim_wheel
+          check_name: TT-Forge-FE Slim Wheel Tests
+          comment: false
+          updateComment: false
+          detailed_summary: true
+          group_suite: true
 
       - name: Run Test
+        if: ${{ inputs.test_mark != 'slim_wheel' }}
         env:
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
           HF_HOME: /mnt/dockercache/huggingface
@@ -235,3 +368,14 @@
           updateComment: false
           detailed_summary: true
           group_suite: true
+
+  build-docker:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Log in to GHCR
+        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u ${{ github.actor }} --password-stdin
+      - name: Build and push docker images if hash changed
+        run: |
+          chmod +x .github/build-docker-images.sh
+          .github/build-docker-images.sh

env/core_requirements.txt

@@ -1,48 +1,27 @@
-setuptools>=61
-# This is needed to avoid issue https://yyz-gitlab.local.tenstorrent.com/devops/devops/-/issues/95
-# jax requires any version of optax which requires any version of chex which in turn
-# requires jax>=0.4.6 which conflicts with our jax == 0.3.16
-# TODO: Remove when jax library is upgraded
-chex==0.1.6
-dataclasses-json==0.5.7
-datasets==2.14.6
-decorator==5.1.1
-flatbuffers==24.3.25
-# This is needed to prevent AttributeError: module 'ml_dtypes' has no attribute 'float8_e4m3b11'
-flax==0.10.6
-jax==0.6.1
-loguru==0.5.3
-networkx==2.8.5
+#core requirements - only essentials for compiler e2e
+Cython
+pybind11==2.6.2
 numpy==1.26.4
+loguru==0.5.3
+
+#framework requirements - core ML frameworks
 onnx>=1.15.0
 onnxruntime>=1.16.3
-opencv-python-headless==4.11.0.86
-# This is needed to avoid issue https://yyz-gitlab.local.tenstorrent.com/devops/devops/-/issues/95
-pandas==1.5.3
-pybind11==2.6.2
-pyinstrument>=4.1.1
-scipy>=1.8.0
-tensorflow==2.19.0
-tensorboard==2.19.0
 tf2onnx==1.15.1
-transformers==4.52.4
-# To avoid warning during the import
-requests==2.28.2
+tensorflow==2.19.0
 tflite==2.10.0
-ultralytics==8.3.91
-paddlepaddle==2.6.2
-paddlenlp==2.8.1
-aistudio-sdk==0.2.6
-pytorch_forecasting==1.0.0
-patool
+torch @ https://download.pytorch.org/whl/cpu/torch-2.7.0%2Bcpu-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version == "3.10"
+
+# needed for unit tests
+pytest==6.2.4
+transformers==4.52.4
+pyinstrument>=4.1.1
+dataclasses-json==0.5.7
 openpyxl==3.1.5
-GitPython==3.1.44
-mlp-mixer-pytorch==0.2.0
-gliner==0.2.7
-ase==3.24.0
-hippynn==0.0.3
-bi-lstm-crf==0.2.1
-peft
-pyclipper==1.3.0
-shapely==2.1.1
-pyarrow==20.0.0
+
+# Paddle stack
+paddlepaddle==2.6.2
+
+# JAX stack
+jax==0.6.1
+flax==0.10.6

env/create_venv.sh

@@ -22,14 +22,8 @@ if [[ -z "$CURRENT_SOURCE_DIR" ]]; then
     exit 1
 fi
 
-# Torch requires a specific version of wheel to be installed
-# which depends on the platform
-if [[ "$OSTYPE" == "darwin"* ]]; then
-    REQUIREMENTS_FILE="$CURRENT_SOURCE_DIR/mac_requirements.txt"
-else
-    # TODO test on linux
-    REQUIREMENTS_FILE="$CURRENT_SOURCE_DIR/linux_requirements.txt"
-fi
+
+REQUIREMENTS_FILE="$CURRENT_SOURCE_DIR/core_requirements.txt"
 
 $TTFORGE_PYTHON_VERSION -m venv $TTFORGE_VENV_DIR
 unset LD_PRELOAD