Refactor qnn-data and remove unneeded dependencies (#113)

* Move testcode to tests and notebook code to notebooks

* Remove imagenet.py from main finn package

* Add missing init py

* Exclude tests folder from docstrings requirement

* Add missing docstrings

* Update imports

* Fix last imports

* Try adding non python files to test package

* Fix missing files

* Add vivado scripts to package

* update toml

* Try fixing missing vivado_scripts

* Debug vivado run failures

* Try fixing imports in notebooks

* Allow copy to not find files

* Fix imports

* New modular packages for tests and notebooks

* Update imports

* Fix building wheels

* Fix poetry path

* Fix poetry paths in .gitlab-ci.yml

* Try to get poetry to finally build the wheels for tests and notebooks correctly

* Try workaround for poetry bug

* Poetry bug workaround

* Test CI corruption

* Try fixing poetry bug withot breaking the pipeline

* Fix notebooks

* Fix vivado project generation

* Start pruning dependencies

* Fix missing wget package in tests

* Update gitignore

* Try excluding torch and dvc from package

* For testing set version fixed to 1.0.1

Uncomment version line and set version to 1.0.1

* Fix version formatting in pyproject.toml

* Add missing package dependency definition for subpackages

* Fix env dir var resolution for notebooks and tests

* Remove qnn-data folder

* Fix path in tests

* Move pybind dep to poetry and clean up dependencies

* Fix fpga module stuff for pipeline

* Fix fpga module stuff for pipeline

* Make sure Python 3.11 is available.

* Fix typo

* Bump version of finn-plus extras in toml

* Revert Python to 3.10 and add pybind for compatibility until fifo rework

* Revert to automatic versioning

* Configure dynamic version correctly

* Use correct build-system settings for submodules

* Set version correctly

* Fix linting

* Fix docstrings

* Add the missing fifos to get_folding_cfg

Author: LinusJungemann

.github/workflows/release.yml

@@ -16,12 +16,42 @@ jobs:
         run: pip install poetry poetry-dynamic-versioning
       - name: Config
         run: poetry config repositories.pypi https://upload.pypi.org/legacy/
-      - name: Build
+      - name: Build subpackages
         run: |
+          # Build tests subpackage
+          cd tests
           poetry install
           poetry build
-      - name: Publish
-        run: poetry publish -r pypi -u __token__ -p ${{ secrets.PYPI_TOKEN }}
+          export VERSION=$(poetry version -s)
+          cd ..
+
+          # Build notebooks subpackage
+          cd notebooks
+          poetry install
+          poetry build
+          cd ..
+
+          # Update main pyproject.toml with the correct version
+          sed -i "s/finn-plus-tests = { version = \"\*\", optional = true }/finn-plus-tests = { version = \"$VERSION\", optional = true }/" pyproject.toml
+          sed -i "s/finn-plus-notebooks = { version = \"\*\", optional = true }/finn-plus-notebooks = { version = \"$VERSION\", optional = true }/" pyproject.toml
+      - name: Build main package
+        run: |
+          poetry install
+          poetry build
+      - name: Publish all packages
+        run: |
+          # Publish tests subpackage
+          cd tests
+          poetry publish -r pypi -u __token__ -p ${{ secrets.PYPI_TOKEN }}
+          cd ..
+
+          # Publish notebooks subpackage
+          cd notebooks
+          poetry publish -r pypi -u __token__ -p ${{ secrets.PYPI_TOKEN }}
+          cd ..
+
+          # Publish main package
+          poetry publish -r pypi -u __token__ -p ${{ secrets.PYPI_TOKEN }}
   test:
     runs-on: ubuntu-24.04
     needs: deploy
@@ -33,4 +63,4 @@ jobs:
       - name: Setup
         run: pip install poetry poetry-dynamic-versioning
       - name: Install
-        run: pip install finn-plus==$(poetry version -s)
+        run: pip install finn-plus[all]==$(poetry version -s)

.gitignore

@@ -42,6 +42,7 @@ __pycache__/*
 *.ipynb_checkpoints*
 *.sif
 .DS_Store
+*.idx
 
 # Project files
 .vscode
@@ -54,6 +55,7 @@ tags
 poetry.lock
 *.code-workspace
 .env
+*/.cache/*
 
 # Package files
 *.egg
@@ -73,6 +75,9 @@ coverage.xml
 build/*
 dist/*
 sdist/*
+*/build/*
+*/dist/*
+*/sdist/*
 docs/api/*
 docs/_rst/*
 docs/_build/*

.gitlab-ci.yml

@@ -99,7 +99,19 @@ Build:
     # Let Poetry create virtual environment (.venv) and install Python dependencies
     # TODO: is it possible to cache the entire venv for the following jobs?
     - poetry install
-    # Build .whl
+    - export POETRY_VIRTUALENVS_CREATE=false #DO NOT TOUCH: needed to build sub-packages
+    # Also build .whl finn-plus-tests package (for test dependencies)
+    - poetry build -P tests --clean --format=wheel
+    # Also build .whl finn-plus-notebooks package (for notebooks dependencies)
+    - poetry build -P notebooks --format=wheel
+    # Set the version of the sub-packages in the main pyproject.toml
+    - |
+      VERSION=$(poetry version -s -C tests)
+      # Update main pyproject.toml with the correct version
+      sed -i "s/finn-plus-tests = { version = \"\*\", optional = true }/finn-plus-tests = { version = \"$VERSION\", optional = true }/" pyproject.toml
+      sed -i "s/finn-plus-notebooks = { version = \"\*\", optional = true }/finn-plus-notebooks = { version = \"$VERSION\", optional = true }/" pyproject.toml
+    - export POETRY_VIRTUALENVS_CREATE=true #DO NOT TOUCH: revert to default behavior
+    # Build .whl finn-plus package
     - poetry build --clean --format=wheel
     # Let FINN download additional dependencies, cache resulting deps directory
     # Also fetch & checkout DVC-managed data, $CI_DVC_CACHE_DIR persists between jobs
@@ -112,6 +124,9 @@ Build:
     name: "finn-plus_package"
     paths:
       - dist/*.whl
+      - tests/dist/*.whl
+      - notebooks/dist/*.whl
+    expire_in: 1 week
   cache:
     key: $CI_COMMIT_SHA #TODO: cache based on relevant files
     paths:

ci/.gitlab-setup.yml

@@ -2,20 +2,17 @@
 
 .n2_setup_general:
   before_script:
-    - module load lang/Python/3.10.4-GCCcore-11.3.0
-    - module load devel/Autoconf/2.71-GCCcore-11.3.0
-    - module load lang/Bison/3.8.2-GCCcore-11.3.0
-    - module load lang/flex/2.6.4-GCCcore-11.3.0
-    - module load lib/pybind11/2.9.2-GCCcore-11.3.0
-    - module load devel/Boost/1.79.0-GCC-11.3.0
-    - module load lib/fmt/9.1.0-GCCcore-11.3.0
-    - module load compiler/GCC/11.3.0
+    #TODO: switch to python 3.11 and remove pybing module (use poetry dependency only) after FiFO sim rework is merged
+    - ml lang/Python/3.10.4-GCCcore-11.3.0
+    - ml lib/pybind11/2.9.2-GCCcore-11.3.0
+    - ml compiler/GCC/14.2.0 devel/Boost/1.88.0-GCC-14.2.0
     - ulimit -s unlimited # Increase stack size limit
 
 .n2_setup_xilinx_2022_2:
   before_script:
     - module load fpga
     - module load xilinx/xrt/2.14 # includes Vitis/Vivado 2022.2
+    - ml fpga/xilinx/vitis_hls_xcd_fix
     - export PLATFORM_PATH=$FINN_CUSTOM_PLATFORM_PATH # override with custom platform path
     # module load will set PLATFORM_REPO_PATHS to one specific platform, revert to top-level PLATFORM_PATH
     - export PLATFORM_REPO_PATHS=$PLATFORM_PATH
@@ -27,6 +24,7 @@
     - module load fpga
     - module load xilinx/xrt/2.14 # includes Vitis/Vivado 2022.2
     - module swap xilinx/vitis/24.2 # switch to Vitis/Vivado 2024.2
+    - ml fpga/xilinx/vitis_hls_xcd_fix
     - export PLATFORM_PATH=$FINN_CUSTOM_PLATFORM_PATH # override with custom platform path
     # module load will set PLATFORM_REPO_PATHS to one specific platform, revert to top-level PLATFORM_PATH
     - export PLATFORM_REPO_PATHS=$PLATFORM_PATH
@@ -40,6 +38,10 @@
     - cd $PATH_WORKDIR
     # Create fresh virtual environment and install finn-plus from .whl (artifact)
     - python3 -m venv finn-plus-venv
+    - finn-plus-venv/bin/pip install 'torch~=2.7.1' 'torchvision~=0.22.1' --extra-index-url https://download.pytorch.org/whl/cu121
+    - finn-plus-venv/bin/pip install 'dvc[webdav]~=3.59.1' 'dvclive[image]~=3.48.2'
+    - finn-plus-venv/bin/pip install tests/dist/*.whl
+    - finn-plus-venv/bin/pip install notebooks/dist/*.whl
     - finn-plus-venv/bin/pip install dist/*.whl
 
 .setup_full_2022_2:

installDependencies.sh

@@ -4,35 +4,10 @@ XRT_DEB_VERSION="xrt_202220.2.14.354_22.04-amd64-xrt"
 
 sudo apt-get update && sudo apt-get install -y \
     build-essential \
-    libc6-dev-i386 \
-    libglib2.0-0 \
-    libsm6 \
-    libxext6 \
-    libxrender-dev \
-    zsh \
     zip \
-    perl \
     make \
-    autoconf \
     g++ \
-    flex \
-    bison \
-    ccache \
-    libgoogle-perftools-dev \
-    numactl \
-    perl-doc \
-    libfl2 \
-    libfl-dev \
-    zlib1g \
     zlib1g-dev \
-    pybind11-dev \
-    libfmt-dev \
-    libboost-dev \
-    libjansson-dev \
-    libgetdata-dev \
-    g++-10 \
-    libpython3.10-dev \
-    python3.10-dev \
     libboost-all-dev
 
 xrt_found=$(dpkg -l | grep xrt | wc -l)

src/finn/qnn-data/__init__.py notebooks/__init__.pysrc/finn/qnn-data/__init__.py renamed to notebooks/__init__.py

@@ -17,7 +17,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from finn.util.visualization import showSrc, showInNetron"
+    "from notebooks.notebook_utils import showSrc, showInNetron"
    ]
   },
   {

notebooks/advanced/0_custom_analysis_pass.ipynb

@@ -17,7 +17,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from finn.util.visualization import showSrc, showInNetron"
+    "from notebooks.notebook_utils import showSrc, showInNetron"
    ]
   },
   {

notebooks/advanced/1_custom_transformation_pass.ipynb

@@ -23,7 +23,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from finn.util.visualization import showInNetron, showSrc"
+    "from notebooks.notebook_utils import showInNetron, showSrc"
    ]
   },
   {

notebooks/advanced/3_folding.ipynb

@@ -61,7 +61,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from finn.util.visualization import showInNetron, showSrc\n",
+    "from notebooks.notebook_utils import showInNetron, showSrc\n",
     "import os\n",
     "    \n",
     "build_dir = os.environ['FINN_NOTEBOOKS'] + \"/advanced\""
@@ -83,7 +83,7 @@
    "outputs": [],
    "source": [
     "import torch\n",
-    "from finn.util.test import get_test_model_trained\n",
+    "from notebooks.notebook_utils import get_test_model_trained\n",
     "from brevitas.export import export_qonnx\n",
     "from qonnx.util.cleanup import cleanup as qonnx_cleanup\n",
     "\n",
@@ -387,7 +387,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from finn.util.pytorch import ToTensor\n",
+    "from notebooks.notebook_utils import ToTensor\n",
     "from qonnx.transformation.merge_onnx_models import MergeONNXModels\n",
     "from qonnx.core.modelwrapper import ModelWrapper\n",
     "from qonnx.core.datatype import DataType\n",
@@ -1350,7 +1350,7 @@
    "outputs": [],
    "source": [
     "# Get golden io pair from Brevitas and save as .npy files\n",
-    "from finn.util.test import get_trained_network_and_ishape, get_example_input, get_topk\n",
+    "from notebooks.notebook_utils import get_trained_network_and_ishape, get_example_input, get_topk\n",
     "import numpy as np\n",
     "\n",
     "\n",