[Deps] Dependency Split

.github/Dockerfile.base

@@ -3,6 +3,7 @@ SHELL ["/bin/bash", "-c"]
 
 # Set environment variables
 ENV DEBIAN_FRONTEND=noninteractive
+ENV PROJECT_NAME=tt-forge-fe
 
 # Install dependencies
 RUN apt-get update && apt-get install -y \
@@ -72,3 +73,32 @@ RUN add-apt-repository ppa:deadsnakes/ppa && \
 # Install python packages
 RUN pip install cmake \
     pytest
+
+# Set environment variables for toolchain directories
+ENV TTFORGE_TOOLCHAIN_DIR=/opt/ttforge-toolchain
+ENV TTMLIR_TOOLCHAIN_DIR=/opt/ttmlir-toolchain
+
+# Create a directory for the build and toolchain
+ARG BUILD_DIR=/home/build
+
+RUN mkdir -p $BUILD_DIR && \
+    mkdir -p $TTMLIR_TOOLCHAIN_DIR && \
+    mkdir -p $TTFORGE_TOOLCHAIN_DIR
+# Create the virtual environment that forge-fe will use
+ENV TTFORGE_VENV_DIR=/opt/ttforge-toolchain/venv
+RUN python3.11 -m venv $TTFORGE_VENV_DIR
+# Copy and install core requirements in the forge-fe virtual environment
+COPY env/core_requirements.txt /tmp/core_requirements.txt
+RUN source $TTFORGE_VENV_DIR/bin/activate && \
+    pip install --no-cache-dir -r /tmp/core_requirements.txt
+
+# Copy the project to the container for building
+ADD . $BUILD_DIR/$PROJECT_NAME
+
+# Build the toolchain
+WORKDIR $BUILD_DIR/$PROJECT_NAME
+RUN source env/activate && \
+    cmake -B env/build env && \
+    cmake --build env/build
+# Ensure venv Python is used by default in derived contexts
+ENV PATH="/opt/ttforge-toolchain/venv/bin:${PATH}"

.github/Dockerfile.ci

@@ -24,6 +24,9 @@ WORKDIR $BUILD_DIR/$PROJECT_NAME
 # Show last commit
 RUN git log -1
 
+# Install dev requirements into the same venv as base using env/activate
+RUN source env/activate && \
+    pip install --no-cache-dir -r env/dev_requirements.txt
 # Build the toolchain
 WORKDIR $BUILD_DIR/$PROJECT_NAME
 RUN source env/activate && \

.github/workflows/build-image.yml

@@ -15,8 +15,11 @@ on:
         type: string
     outputs:
       docker-image:
-        description: "Built docker image name"
+        description: "Built CI docker image name"
         value: ${{ jobs.build-image.outputs.docker-image }}
+      base-image:
+        description: "Built base IRD docker image name"
+        value: ${{ jobs.build-image.outputs.base-image }}
 
 
 permissions:
@@ -29,6 +32,7 @@ jobs:
     runs-on: tt-ubuntu-2204-large-stable
     outputs:
       docker-image: ${{ steps.build.outputs.docker-image }}
+      base-image: ${{ steps.build.outputs.base-image }}
     steps:
       - name: Fix permissions
         shell: bash
@@ -73,12 +77,16 @@ jobs:
         id: build
         shell: bash
         run: |
-          # Output the image name
+          # Output the image names
           set pipefail
           .github/build-docker-images.sh | tee docker.log
-          DOCKER_CI_IMAGE=$(tail -n 1 docker.log)
+          DOCKER_TAG=$(./.github/get-docker-tag.sh)
+          DOCKER_CI_IMAGE="ghcr.io/tenstorrent/tt-forge-fe/tt-forge-fe-ird-ubuntu-22-04:$DOCKER_TAG"
+          DOCKER_BASE_IMAGE="ghcr.io/tenstorrent/tt-forge-fe/tt-forge-fe-base-ubuntu-22-04:$DOCKER_TAG"
           echo "DOCKER_CI_IMAGE $DOCKER_CI_IMAGE"
+          echo "DOCKER_BASE_IMAGE $DOCKER_BASE_IMAGE"
           echo "docker-image=$DOCKER_CI_IMAGE" >> "$GITHUB_OUTPUT"
+          echo "base-image=$DOCKER_BASE_IMAGE" >> "$GITHUB_OUTPUT"
 
   set-latest:
     # Set the latest tag on the IRD image

.github/workflows/on-pr.yml

@@ -53,6 +53,20 @@ jobs:
     secrets: inherit
     with:
       docker-image: ${{ needs.docker-build.outputs.docker-image }}
+  test-slim:
+    needs:
+      - docker-build
+      - build
+    uses: ./.github/workflows/test-sub.yml
+    secrets: inherit
+    with:
+      test_mark: 'slim'
+      test_group_cnt: 1
+      test_group_ids: '[1]'
+      docker-image: ${{ needs.docker-build.outputs.base-image }}
+      run_id: ${{ github.run_id }}
+      runs-on: '[{"runs-on": "n150"}, {"runs-on": "n300"}]'
+
   test:
     needs:
       - docker-build
@@ -95,6 +109,7 @@ jobs:
       - docker-build
       - build-docs
       - build
+      - test-slim
       - test
       - perf-benchmark
     runs-on: Ubuntu-latest

conftest.py

@@ -13,7 +13,13 @@
     ForgePropertyHandler,
     forge_property_handler_var,
 )
-from forge._C.verif import malloc_trim
+
+try:
+    from forge._C.verif import malloc_trim
+except ImportError:
+    # For test-slim, we don't have the C extensions
+    def malloc_trim():
+        pass
 
 
 def pytest_addoption(parser):

env/core_requirements.txt

@@ -1,48 +1,8 @@
-setuptools>=61
-# This is needed to avoid issue https://yyz-gitlab.local.tenstorrent.com/devops/devops/-/issues/95
-# jax requires any version of optax which requires any version of chex which in turn
-# requires jax>=0.4.6 which conflicts with our jax == 0.3.16
-# JAX 0.7 compatible dependencies
-chex>=0.1.7
-dataclasses-json==0.5.7
-datasets==2.14.6
-decorator==5.1.1
-flatbuffers==24.3.25
-# This is needed to prevent AttributeError: module 'ml_dtypes' has no attribute 'float8_e4m3b11'
-flax>=0.10.6
-jax==0.7.1
-loguru==0.5.3
-networkx==2.8.5
+#core requirements - only essentials for compiler e2e
 numpy==1.26.4
-onnx>=1.15.0
-onnxruntime>=1.16.3
-opencv-python-headless==4.11.0.86
-# This is needed to avoid issue https://yyz-gitlab.local.tenstorrent.com/devops/devops/-/issues/95
-pandas==1.5.3
-pybind11==2.6.2
-pyinstrument>=4.1.1
-scipy>=1.8.0
-tensorflow==2.19.0
-tensorboard==2.19.0
-tf2onnx==1.15.1
-transformers==4.52.4
-# To avoid warning during the import
-requests==2.28.2
-tflite==2.10.0
-ultralytics==8.3.91
-paddlepaddle==2.6.2
-paddlenlp==2.8.1
-aistudio-sdk==0.2.6
-pytorch_forecasting
-patool
-openpyxl==3.1.5
-GitPython==3.1.44
-mlp-mixer-pytorch==0.2.0
-gliner==0.2.7
-ase==3.24.0
-hippynn==0.0.3
-bi-lstm-crf==0.2.1
-peft
-pyclipper>=1.3.0
-shapely==2.1.1
-pyarrow==20.0.0
+loguru==0.5.3
+psutil==5.9.8
+pybind11>=2.10.0
+pytest>=8.0.0
+torch @ https://download.pytorch.org/whl/cpu/torch-2.7.0%2Bcpu-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version == "3.11"
+torchvision @ https://download.pytorch.org/whl/cpu/torchvision-0.22.0%2Bcpu-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version == "3.11"

env/create_venv.sh

@@ -28,7 +28,7 @@ if [[ "$OSTYPE" == "darwin"* ]]; then
     REQUIREMENTS_FILE="$CURRENT_SOURCE_DIR/mac_requirements.txt"
 else
     # TODO test on linux
-    REQUIREMENTS_FILE="$CURRENT_SOURCE_DIR/linux_requirements.txt"
+    REQUIREMENTS_FILE="$CURRENT_SOURCE_DIR/dev_requirements.txt"
 fi
 
 $TTFORGE_PYTHON_VERSION -m venv $TTFORGE_VENV_DIR

env/dev_requirements.txt

@@ -0,0 +1,135 @@
+#dev-requirements
+# This is needed to avoid issue https://yyz-gitlab.local.tenstorrent.com/devops/devops/-/issues/95
+# jax requires any version of optax which requires any version of chex which in turn
+# requires jax>=0.4.6 which conflicts with our jax == 0.3.16
+
+
+# JAX Stack (JAX 0.7 compatible dependencies)
+
+jax==0.7.1
+chex>=0.1.7
+# This is needed to prevent AttributeError: module 'ml_dtypes' has no attribute 'float8_e4m3b11'
+flax>=0.10.6
+
+
+# Unit Testing Framework
+
+pytest==6.2.4
+pytest-timeout==2.0.1
+pytest-xdist==2.5.0
+pytest-split
+
+
+# Deep Learning Frameworks
+
+tensorflow==2.19.0
+tensorboard==2.19.0
+transformers==4.52.4
+diffusers==0.32.1
+
+
+# PaddlePaddle Stack
+
+paddlepaddle==2.6.2
+paddlenlp==2.8.1
+aistudio-sdk==0.2.6
+
+
+# ONNX Ecosystem
+
+onnx>=1.15.0
+onnxruntime>=1.16.3
+tf2onnx==1.15.1
+tflite==2.10.0
+
+
+# Computer Vision Libraries
+
+opencv-python-headless==4.11.0.86
+opencv-contrib-python==4.9.0.80
+timm==1.0.9
+ultralytics==8.3.91
+yolov5==7.0.9
+yolov6detect==0.4.1
+pytorchcv==0.0.67
+segmentation_models_pytorch==0.4.0
+torchxrayvision==0.0.39
+vgg_pytorch==0.3.0
+# For DenseNet 121 HF XRay model
+scikit-image==0.20.0
+# Required by the RMBG model
+kornia
+
+
+# Data Processing & Datasets
+
+# This is needed to avoid issue https://yyz-gitlab.local.tenstorrent.com/devops/devops/-/issues/95
+pandas==1.5.3
+datasets==2.14.6
+pyarrow==20.0.0
+openpyxl==3.1.5
+
+
+# Audio Processing
+
+# Required by the Whisper model package for audio processing
+librosa
+soundfile
+
+
+# Model Fine-tuning & Optimization
+
+peft==0.15.1
+
+
+# Specialized ML Models
+
+mlp-mixer-pytorch==0.2.0
+gliner==0.2.7
+pytorch_forecasting
+bi-lstm-crf==0.2.1
+# Atomic simulation environment
+ase==3.24.0
+hippynn==0.0.3
+
+
+# Geometry & Image Processing Utilities
+
+pyclipper>=1.3.0
+shapely==2.1.1
+
+
+# Scientific Computing & Utilities
+
+scipy>=1.8.0
+networkx==2.8.5
+decorator==5.1.1
+seaborn
+
+
+# Development & Debugging Tools
+
+pyinstrument>=4.1.1
+clang-format==18.1.7
+tabulate==0.9.0
+
+
+# Serialization & Data Formats
+
+dataclasses-json==0.5.7
+flatbuffers==24.3.25
+safetensors==0.6.0.dev0
+
+
+# System & Build Dependencies
+
+pybind11==2.6.2
+# To avoid warning during the import
+requests==2.28.2
+GitPython==3.1.44
+python-gitlab==4.4.0
+patool
+
+#to resolve cuda Error - CUDA toolkit was not found or specified
+torch @ https://download.pytorch.org/whl/cpu/torch-2.7.0%2Bcpu-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version == "3.11"
+torchvision @ https://download.pytorch.org/whl/cpu/torchvision-0.22.0%2Bcpu-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version == "3.11"