Merge dev into main

.github/workflows/automerge.yml

@@ -19,9 +19,9 @@ on:
   status: {}
 jobs:
   automerge:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - name: automerge
-        uses: "pascalgn/automerge-action@v0.12.0"
+        uses: "pascalgn/automerge-action@v0.16"
         env:
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/create_pull_request.yml

@@ -7,15 +7,15 @@ jobs:
   create_pull_request:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: main
       - name: Reset main branch with dev changes
         run: |
           git fetch origin dev:dev
           git reset --hard dev
       - name: Create pull request from dev to main
-        uses: peter-evans/create-pull-request@v3
+        uses: peter-evans/create-pull-request@v7
         with:
           token: ${{ secrets.GITHUBACTIONS }} 
           title: Merge dev into main

.github/workflows/lint.yml

@@ -12,31 +12,29 @@ jobs:
     strategy:
       matrix:
         python-version: 
-          - "3.10"
+          - "3.12"
     steps:
     - uses: actions/cache@v4
       with:
         path: ~/.cache/pip
         key: ${{ runner.os }}-pip-${{ hashFiles('**/pyproject.toml') }}
         restore-keys: |
           ${{ runner.os }}-pip-
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        curl -sSL https://raw.githubusercontent.com/legendu-net/icon/main/install_icon.sh | sudo bash -
-        icon pt -ic
-        ~/.local/bin/poetry env use python3
-        ~/.local/bin/poetry install -E all
-    - name: Lint with pylint
+        curl -LsSf https://astral.sh/uv/install.sh | sh
+        ~/.local/bin/uv sync --all-extras
+    - name: Lint with ruff
       run: |
-        ~/.local/bin/poetry run pylint -E aiutil/ tests/
-    # - name: Lint with pytype
-    #   run: |
-    #     ~/.local/bin/poetry run pytype ./
+        ~/.local/bin/uv run ruff check aiutil/ tests/
+    - name: Lint with pytype
+      run: |
+        ~/.local/bin/uv run pytype ./
     - name: Check code format
       run: |
-        ~/.local/bin/poetry run black --check ./
+        ~/.local/bin/uv run ruff format --check .

.github/workflows/pre-release.yml

@@ -12,18 +12,17 @@ jobs:
       pull-requests: write
       contents: write
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
-      - name: Install Poetry
+          python-version: "3.12"
+      - name: Install uv
         run: |
-          curl -sSL https://raw.githubusercontent.com/legendu-net/icon/main/install_icon.sh | sudo bash -
-          icon pt -ic
+          curl -LsSf https://astral.sh/uv/install.sh | sh
       - name: Build Package
         run: |
-          ~/.local/bin/poetry build
+          ~/.local/bin/uv build
           ls -lha dist/
       - uses: "marvinpinto/action-automatic-releases@latest"
         with:

.github/workflows/release.yml

@@ -12,19 +12,19 @@ jobs:
       pull-requests: write
       contents: write
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
-      - name: Install Poetry
+          python-version: "3.12"
+      - name: Install uv
         run: |
-          curl -sSL https://raw.githubusercontent.com/legendu-net/icon/main/install_icon.sh | sudo bash -
-          icon pt -ic
+          curl -LsSf https://astral.sh/uv/install.sh | sh
       - name: Build and Pubblish Package to PyPI
         run: |
-          ~/.local/bin/poetry publish --build -u __token__ -p ${{ secrets.PYPI_AIUTIL }}
+          ~/.local/bin/uv build
           ls -lha dist/
+          ~/.local/bin/uv publish -u __token__ -p ${{ secrets.PYPI_AIUTIL }}
       - uses: "marvinpinto/action-automatic-releases@latest"
         with:
           repo_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/test_macos.yml

@@ -15,22 +15,21 @@ jobs:
         python-version: 
           - "3.10"
           - "3.11"
+          - "3.12"
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
         curl -sSL https://raw.githubusercontent.com/legendu-net/icon/main/install_icon.sh | sudo bash -
-        icon docker -ic --user-to-docker ""
-        icon pt -ic
-        ~/.local/bin/poetry lock -n
-        ~/.local/bin/poetry env use python3
-        ~/.local/bin/poetry install -E all
+        #icon docker -ic --user-to-docker ""
+        curl -LsSf https://astral.sh/uv/install.sh | sh
+        ~/.local/bin/uv sync
         brew install libmagic
         brew link libmagic
     - name: Test with pytest
       run: |
-        ~/.local/bin/poetry run pytest
+        ~/.local/bin/uv run pytest

.github/workflows/test_ubuntu.yml

@@ -14,19 +14,17 @@ jobs:
         python-version: 
           - "3.10"
           - "3.11"
+          - "3.12"
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        curl -sSL https://raw.githubusercontent.com/legendu-net/icon/main/install_icon.sh | sudo bash -
-        icon pt -ic
-        ~/.local/bin/poetry lock -n
-        ~/.local/bin/poetry env use python3
-        ~/.local/bin/poetry install -E all
+        curl -LsSf https://astral.sh/uv/install.sh | sh
+        ~/.local/bin/uv sync
     - name: Test with pytest
       run: |
-        ~/.local/bin/poetry run pytest
+        ~/.local/bin/uv run pytest

.idx/dev.nix

@@ -1,13 +1,15 @@
 {pkgs, ...}: {
   channel = "stable-24.11";
   packages = with pkgs; [
+    util-linux
     bash-completion
     gitui
     neovim
     ripgrep
     rm-improved
     bat
-    poetry
+    fzf
+    uv
   ];
   env = {
     PATH = [
@@ -25,11 +27,7 @@
       #onCreate = {
       #}
       onStart = {
-        poetry-project = ''
-        poetry config --local virtualenvs.in-project true
-        poetry install
-        '';
-          icon = ''
+        icon = ''
         curl -sSL https://raw.githubusercontent.com/legendu-net/icon/main/install_icon.sh | bash -s -- \
             -d ~/.local/bin
         '';

aiutil/__init__.py

@@ -1,5 +1,3 @@
 """A utils Python package for data scientists."""
 
-from . import poetry
-
-__version__ = "0.83.0"
+__version__ = "0.84.0"

aiutil/filesystem.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 """Filesystem related util functions."""
+
 from typing import Iterable, Callable
 import itertools
 import os
@@ -548,9 +549,10 @@ def dedup_header(file: str | Path, output: str | Path = "") -> None:
     :param output: The path of the output file.
         If empty, then output to the standard output.
     """
-    with open(file, "rb") as fin, (
-        open(output, "wb") if output else sys.stdout.buffer
-    ) as fout:
+    with (
+        open(file, "rb") as fin,
+        open(output, "wb") if output else sys.stdout.buffer as fout,
+    ):
         header = fin.readline()
         fout.write(header)
         for line in fin:
@@ -606,9 +608,10 @@ def prune_json(input: str | Path, output: str | Path = ""):
         else:
             output = input.with_name(input.stem + "_prune.json")
     skip = False
-    with input.open("r", encoding="utf-8") as fin, output.open(
-        "w", encoding="utf-8"
-    ) as fout:
+    with (
+        input.open("r", encoding="utf-8") as fin,
+        output.open("w", encoding="utf-8") as fout,
+    ):
         for line in fin:
             line = line.strip()
             if line == '"value_counts": {':

aiutil/hadoop/__init__.py

@@ -2,3 +2,5 @@
 
 from .hdfs import Hdfs
 from .log import LogFilter
+
+__all__ = ["Hdfs", "LogFilter"]

aiutil/hadoop/kerberos.py

@@ -5,6 +5,7 @@
 If an user authenticate without specifying password,
 the saved password is used so that users do not have to type in password to authenticate every time.
 """
+
 from typing import Any
 import os
 from pathlib import Path
@@ -81,7 +82,7 @@ def authenticate(password: str, email: dict[str, str], user: str = "") -> None:
     :param user: User name. If empty, the current user name is used.
     """
     SUBJECT = "kinit: authentication {}"
-    MSG = f'kinit ({PID}): authentication on {HOST} ({HOST_IP}) {"{}"} at {datetime.datetime.now()}'
+    MSG = f"kinit ({PID}): authentication on {HOST} ({HOST_IP}) {'{}'} at {datetime.datetime.now()}"
     try:
         process = sp.run(
             ["/usr/bin/kinit", user if user else USER],

aiutil/hadoop/pyspark_submit.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 # encoding: utf-8
 """A module makes it easy to run Scala/Python Spark job."""
+
 from typing import Callable, Any, Iterable
 import os
 import sys
@@ -31,7 +32,7 @@ def __init__(self, email: dict | None = None, level: str = "INFO"):
         # set up loguru with the right logging level
         try:
             logger.remove(0)
-        except:
+        except Exception:
             pass
         logger.add(sys.stdout, level=level)
         self._spark_submit_log = {}

aiutil/hash.py

@@ -35,7 +35,7 @@ def _rmd5(path: Path, res: list[str]) -> None:
     if path.is_file():
         try:
             md5sum = hashlib.md5(path.read_bytes()).hexdigest()
-        except:
+        except Exception:
             md5sum = "FAILED!"
         line = f"{str(path)}: {md5sum}"
         res.append(line)
@@ -44,7 +44,7 @@ def _rmd5(path: Path, res: list[str]) -> None:
     try:
         for p in path.iterdir():
             _rmd5(p, res)
-    except:
+    except Exception:
         line = f"{str(path)}: FAILED!"
         res.append(line)
         logger.info(line)

aiutil/memory.py

@@ -34,7 +34,7 @@ def get_memory_usage(user: str = USER) -> int:
             for p in psutil.process_iter()
             if p.username() == USER and p.status() in STATUS
         )
-    except:
+    except Exception:
         return get_memory_usage(user)
 
 

aiutil/notebook/search.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 """Search Jupyter/Lab notebooks."""
+
 from typing import TypeAlias, Sequence
 import json
 from pathlib import Path
@@ -94,8 +95,8 @@ def match_language(self, language: Criterion) -> bool:
         :param language: A language criterion.
         """
         lang = _reg_criterion(language)
-        return all(self.lang == l.lower() for l in lang["include"]) and not any(
-            self.lang == l.lower() for l in lang["exclude"]
+        return all(self.lang == lg.lower() for lg in lang["include"]) and not any(
+            self.lang == lg.lower() for lg in lang["exclude"]
         )
 
     def cells(self, keyword: Criterion, type_: str = "") -> list[Cell]:

aiutil/notebook/util.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 """Jupyter/Lab notebooks related utils."""
+
 from pathlib import Path
 import subprocess as sp
 import itertools as it
@@ -46,7 +47,7 @@ def _find_path_content(path, pattern):
             try:
                 if pattern in p.read_text():
                     yield p
-            except:
+            except Exception:
                 pass