Merge pull request #1079 from onekey-sec/uv

Poetry -> uv

Author: vlaci

.github/ISSUE_TEMPLATE/bug-report-🐞.md

@@ -46,14 +46,14 @@ A clear and concise description of what you expected to happen.
 
 **Environment information (please complete the following information):**
  - OS: [e.g. Ubuntu Linux]
- - Software versions (Docker, Python, Nix, Poetry, if applicable)
+ - Software versions (Docker, Python, Nix, Uv, if applicable)
 
 We recommend you execute and paste the results of those commands in this issue so we can get a sense of your environment:
 
 - Linux/Darwin kernel version with `uname -avr`
 - Linux distribution with `cat /etc/lsb-release`
 - MacOS distribution with `sw_vers`
-- Poetry version with `poetry env info`
+- Uv version with `uv version`
 - Nix install info with `nix flake info`
 - Pip install info `pip freeze`
 - Unblob dependencies info with `unblob --show-external-dependencies`

.github/actions/setup-dependencies/action.yml

@@ -13,35 +13,13 @@ runs:
       run: sudo unblob/install-deps.sh
       shell: bash
 
-    - name: Setup Python
-      uses: actions/setup-python@v5
+    - name: Install uv & Python
+      uses: astral-sh/setup-uv@v5
       with:
-        python-version: ${{ inputs.python-version }}
+        enable-cache: true
+        cache-dependency-glob: "uv.lock"
+        python-version: ${{ matrix.python-version }}
 
-    - name: Restore pip cache
-      uses: actions/cache@v4
-      with:
-        path: ~/.cache/pip
-        key: pip-${{ inputs.python-version }}
-
-    - name: Pip install
-      run: python -m pip install --upgrade pip poetry
-      shell: bash
-
-    - name: Restore poetry cache
-      uses: actions/cache@v4
-      with:
-        path: |
-          ~/.cache/pypoetry/cache
-          ~/.cache/pypoetry/artifacts
-        key: poetry-cache-and-artifacts-${{ inputs.python-version }}
-
-    - name: Restore virtualenvs
-      uses: actions/cache@v4
-      with:
-        path: ~/.cache/pypoetry/virtualenvs
-        key: venv-${{ hashFiles('poetry.lock') }}-${{ inputs.python-version }}
-
-    - name: Poetry install
-      run: poetry install --with dev
+    - name: Install the project
+      run: uv sync
       shell: bash

.github/workflows/build-publish-image.yml

@@ -34,8 +34,8 @@ jobs:
       - name: Setup 3rd party dependencies
         uses: ./.github/actions/setup-dependencies
 
-      - name: Poetry build
-        run: poetry build --format wheel
+      - name: uv build
+        run: uv build --wheel
 
       - name: Docker meta
         id: meta

.github/workflows/documentation.yml

@@ -13,43 +13,15 @@ jobs:
       - name: Checkout source code
         uses: actions/checkout@v4
 
-      - name: Setup Python
-        uses: actions/setup-python@v5
+      - name: Install uv & Python
+        uses: astral-sh/setup-uv@v5
         with:
-          python-version: 3.9
+          enable-cache: true
+          cache-dependency-glob: "uv.lock"
+          python-version: "3.9"
 
-      - name: Setup Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: 3.9
-
-      - name: Restore pip cache
-        uses: actions/cache@v4
-        with:
-          path: ~/.cache/pip
-          key: pip-3.9
-
-      - name: Upgrade pip and install poetry
-        run: python -m pip install --upgrade pip poetry
-        shell: bash
-
-      - name: Restore poetry cache
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/pypoetry/cache
-            ~/.cache/pypoetry/artifacts
-          key: poetry-cache-and-artifacts-3.9
-
-      - name: Restore virtualenvs
-        uses: actions/cache@v4
-        with:
-          path: ~/.cache/pypoetry/virtualenvs
-          key: venv-${{ hashFiles('poetry.lock') }}-3.9
-
-      - name: Poetry install
-        run: poetry install --only docs
-        shell: bash
+      - name: Install the project
+        run: uv sync --only-group docs
 
       - name: Generate documentation
-        run: poetry run mkdocs gh-deploy --force
+        run: uv run mkdocs gh-deploy --force

.github/workflows/release.yml

@@ -15,6 +15,7 @@ jobs:
       - name: Setup dependencies
         uses: ./.github/actions/setup-dependencies
 
-      - run: poetry publish --build
+      - run: uv build
+      - run: uv publish
         env:
-          POETRY_PYPI_TOKEN_PYPI: ${{secrets.POETRY_PYPI_TOKEN_PYPI}}
+          UV_PUBLISH_TOKEN: ${{secrets.POETRY_PYPI_TOKEN_PYPI}}

.github/workflows/run-tests.yml

@@ -41,7 +41,7 @@ jobs:
         uses: ./.github/actions/setup-dependencies
 
       - name: Check - pyright
-        run: poetry run pyright .
+        run: uv run pyright .
 
   run_tests:
     name: Run tests
@@ -62,4 +62,4 @@ jobs:
         uses: ./.github/actions/setup-git-lfs
 
       - name: Run pytest
-        run: poetry run pytest -vvv
+        run: uv run pytest -vvv

.gitignore

@@ -7,7 +7,7 @@ build/
 *.so
 .idea
 .coverage*
-/.venv/
+/.venv
 unblob.log
 /.devenv
 /.direnv

.pre-commit-config.yaml

@@ -20,15 +20,15 @@ repos:
         language: system
         types_or: [python, pyi]
         require_serial: true
-        entry: poetry run ruff check --force-exclude
+        entry: uv run ruff check --force-exclude
         args:
           [--output-format=full, --fix, --show-fixes, --exit-non-zero-on-fix]
       - id: ruff-format
         name: Format python (ruff)
         language: system
         types_or: [python, pyi]
         require_serial: true
-        entry: poetry run ruff format --force-exclude
+        entry: uv run ruff format --force-exclude
       - id: nixfmt
         name: Format Nix (nixfmt)
         types: [nix]
@@ -44,12 +44,12 @@ repos:
         name: Format TOML (taplo)
         language: system
         types: [toml]
-        entry: poetry run taplo format
+        entry: uv run taplo format
       - id: taplo-lint
         name: Lint TOML (taplo)
         language: system
         types: [toml]
-        entry: poetry run taplo lint
+        entry: uv run taplo lint
         args:
           - --default-schema-catalogs
 

Dockerfile

@@ -9,7 +9,7 @@ WORKDIR /data/output
 COPY unblob/install-deps.sh /
 RUN sh -xeu /install-deps.sh
 
-# You MUST do a poetry build before to have the wheel to copy & install here (CI action will do this when building)
+# You MUST do an uv build before to have the wheel to copy & install here (CI action will do this when building)
 COPY dist/*.whl /tmp/
 RUN pip --disable-pip-version-check install --upgrade pip
 RUN pip install /tmp/unblob*.whl --prefix /usr/local

devenv.nix

@@ -1,4 +1,9 @@
-{ lib, pkgs, ... }:
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
 
 # https://devenv.sh/reference/options/
 {
@@ -7,33 +12,49 @@
     libraries = with pkgs; [
       file # python-magic
     ];
-    poetry = {
+    venv.enable = true; # put venv in PATH
+    uv = {
       enable = true;
-      activate.enable = true;
-      install.enable = true;
-      install = {
-        groups = [ "dev" ];
-        installRootPackage = true;
-      };
+      sync.enable = true;
+      sync.arguments = [
+        # by default it contains `--no-install-workspace`
+        "--frozen"
+        "--group"
+        "dev"
+      ];
     };
   };
 
+  env.UV_LINK_MODE = "copy";
+
   packages =
     with pkgs;
     [
-      nvfetcher
       nodejs # for pyright and renovate
     ]
     ++ unblob.runtimeDeps;
 
   tasks = {
+    "venv:link" = {
+      exec = ''
+        VENV_DIR="${config.devenv.root}/.venv"
+
+        if [[ -d "$VENV_DIR" && ! -L "$VENV_DIR" ]]; then
+          echo "Found an existing ${config.devenv.root}/.venv directory. Please remove it."
+          exit 1
+        fi
+        ln -snf "${config.devenv.state}/venv" "${config.devenv.root}/.venv"
+      '';
+      after = [ "devenv:python:uv" ];
+      before = [ "devenv:enterShell" ];
+    };
     "venv:patchelf" = {
       exec = ''
         for exe in taplo ruff; do
-          ${lib.getExe pkgs.patchelf} --set-interpreter ${pkgs.stdenv.cc.bintools.dynamicLinker} $VIRTUAL_ENV/bin/$exe
+          ${lib.getExe pkgs.patchelf} --set-interpreter ${pkgs.stdenv.cc.bintools.dynamicLinker} "${config.devenv.state}/venv/bin/$exe"
         done
       '';
-      after = [ "devenv:python:poetry" ];
+      after = [ "devenv:python:uv" ];
       before = [ "devenv:enterShell" ];
     };
   };

docs/development.md

@@ -32,8 +32,8 @@ where the exciting stuff is.
 - **git**: You need it for cloning the repository.
   Install it [from the git-scm website](https://git-scm.com/download).
 
-- **Poetry**: it is a package manager for Python dependencies. Follow the instructions on the
-  [Poetry website](https://python-poetry.org/docs/master/#installation) to install the latest version.
+- **uv**: it is a package manager for Python dependencies. Follow the instructions on the
+  [uv website](https://docs.astral.sh/uv/getting-started/installation/) to install the latest version.
 
 - **pre-commit**: We are using [pre-commit](https://pre-commit.com/) to run
   checks like linters, type checks and formatting issues.
@@ -72,16 +72,10 @@ You need to setup Git LFS once, before you will be able to run the whole test su
 
 The recommended way to develop Python projects in a semi-isolated way is to use `virtualenv`.
 
-If you don't want to manage it separately, you can rely on `Poetry` to automatically
+If you don't want to manage it separately, you can rely on `uv` to automatically
 create a virtualenv for you on install.
 
-If you don't want Poetry to automatically create a virtualenv, you can turn it off with this command:
-
-```shell
-poetry config virtualenvs.create false
-```
-
-Or instead of Poetry you can use `pyenv`. You can set the Python interpreter
+Or instead of uv you can use `pyenv`. You can set the Python interpreter
 version for the local folder only with:
 
 ```
@@ -90,29 +84,29 @@ pyenv local 3.12.7
 
 ### Installing Python dependencies
 
-We are using [poetry](https://python-poetry.org/) to manage our Python
+We are using [uv](https://docs.astral.sh/uv/) to manage our Python
 dependencies. To install all required dependencies for development, you can run
 the following command:
 
 ```
-poetry install --with dev
+uv install
 ```
 
 Please note that it installs dependencies within the dedicated virtual
 environment. So if you want to run `unblob` or `pytest`, you need to do it from
 within the virtual environment:
 
-Using poetry run:
+Using uv run:
 
 ```
-poetry run unblob
-poetry run pytest tests -v
+uv run unblob
+uv run pytest tests -v
 ```
 
 By dropping into the virtual environment:
 
 ```
-poetry shell
+uv run $SHELL
 unblob
 pytest tests -v
 ```

docs/extractors.md

@@ -49,7 +49,7 @@ The following executables found installed, which are needed by unblob:
 
 ✅: These extractors come with unblob, check
 [pyproject.toml](https://github.com/onekey-sec/unblob/blob/main/pyproject.toml)
-and [poetry.lock](https://github.com/onekey-sec/unblob/blob/main/poetry.lock)
+and [uv.lock](https://github.com/onekey-sec/unblob/blob/main/uv.lock)
 for current versions.
 
 | Extractor                   | Provided commands                                     | Minimum version | Pre-Installed | More information                                                 |

docs/index.md

@@ -60,7 +60,7 @@ unblob has been developed with the following objectives in mind:
   tested](https://github.com/onekey-sec/unblob/issues?q=label%3Afuzzing+)
   against a large corpus of files and firmware images. We rely on up-to-date
   third party dependencies that are
-  [locked](https://github.com/onekey-sec/unblob/blob/main/poetry.lock) to limit
+  [locked](https://github.com/onekey-sec/unblob/blob/main/uv.lock) to limit
   potential supply chain issues. We use safe extractors that we audited and
   fixed where required (see [path traversal in
   ubi_reader](https://github.com/onekey-sec/ubi_reader/commit/4a81f3f0a714bb83d6ee71db09b7748619fa9fb7),

docs/installation.md

@@ -192,23 +192,23 @@ For imperative package installation, follow these steps:
 ## From source
 
 1.  Install [Git](https://git-scm.com/download/) if you don't have it yet.
-2.  Install the [Poetry](https://python-poetry.org/docs/#installation) Python package manager.
+2.  Install [uv](https://docs.astral.sh/uv/getting-started/installation/) Python package manager.
 3.  **Clone** the unblob **repository from GitHub**:
 
         git clone https://github.com/onekey-sec/unblob.git
 
-4.  Install **Python dependencies** with Poetry:
+4.  Install **Python dependencies** with uv:
 
     1.  Python packages:
 
             cd unblob
-            poetry install --no-dev
+            uv sync --no-dev
 
     2.  Make sure you [installed all extractors](#install-extractors).
 
     3.  Check that everything works correctly:
 
-            poetry run unblob --show-external-dependencies
+            uv run unblob --show-external-dependencies
 
 ## Install extractors