Release v0.13.0 (#104)

Author: kurtmckee

.github/workflows/test.yaml

@@ -40,7 +40,6 @@ jobs:
               - "3.13"
             cpython-beta: "3.14"
             pypys:
-              - "3.10"
               - "3.11"
             cache-key-hash-files:
               - "pyproject.toml"

.pre-commit-config.yaml

@@ -25,7 +25,7 @@ repos:
       - id: "trailing-whitespace"
 
   - repo: "https://github.com/asottile/pyupgrade"
-    rev: "v3.19.1"
+    rev: "v3.20.0"
     hooks:
       - id: "pyupgrade"
         name: "Enforce Python 3.9+ idioms"
@@ -47,7 +47,7 @@ repos:
       - id: "isort"
 
   - repo: "https://github.com/pycqa/flake8"
-    rev: "7.2.0"
+    rev: "7.3.0"
     hooks:
       - id: "flake8"
         additional_dependencies:
@@ -59,7 +59,7 @@ repos:
       - id: "editorconfig-checker"
 
   - repo: "https://github.com/python-jsonschema/check-jsonschema"
-    rev: "0.33.0"
+    rev: "0.33.2"
     hooks:
       - id: "check-dependabot"
       - id: "check-readthedocs"

CHANGELOG.rst

@@ -25,6 +25,56 @@ Please see the fragment files in the `changelog.d directory`_.
 
 ..  scriv-insert-here
 
+.. _changelog-0.13.0:
+
+0.13.0 - 2025-07-24
+===================
+
+Python support
+--------------
+
+*   Drop support for PyPy 3.10.
+
+Fixed
+-----
+
+*   Fall back to importing from source code if importing from byte code fails.
+
+    This resolves a problem importing Flask when byte-compiled,
+    due to its ``sansio`` subdirectory, which has no ``__init__.py`` file
+    and whose submodules currently fail to import from the byte code table.
+
+Documentation
+-------------
+
+*   Document compatible ruff configurations.
+
+*   Add a logo and description to the sidebar on subpages.
+
+*   Update the homepage sidebar layout and design.
+
+Development
+-----------
+
+*   Fix performance testing issues.
+
+    Zip-based bytecode import times were skewed during testing
+    because ``zipimport`` doesn't use PEP 3147 ``__pycache__/`` subdirectories.
+    This is now accounted for by the performance testing script's setup steps,
+    and zip-based import times are now accurate for comparison.
+
+    Also, the total size of the source code and byte code trees is captured.
+
+*   Begin to expand and better automate performance testing.
+
+    This includes the ability to install from a ``requirements.txt`` file,
+    to execute an arbitrary Python file, and to plot comparison bar charts
+    of import times and package content sizes.
+
+*   Use chipshot to standardize headers.
+
+*   Prefer the new Python 3.14 ``compression.lzma`` namespace.
+
 .. _changelog-0.12.0:
 
 0.12.0 - 2025-05-15

assets/generate-perftest-directory.py

@@ -0,0 +1,50 @@
+# This file is a part of sqliteimport <https://github.com/kurtmckee/sqliteimport>
+# Copyright 2024-2025 Kurt McKee <[email protected]>
+# SPDX-License-Identifier: MIT
+
+import enum
+import pathlib
+
+REPO_ROOT = pathlib.Path(__file__).parent.parent.parent
+PACKAGE_DIRECTORY = REPO_ROOT / "build/perftest"
+STATS = REPO_ROOT / "build/perfstats"
+
+
+class Importer(enum.StrEnum):
+    filesystem = "filesystem"
+    zipimport = "zipimport"
+    sqliteimport = "sqliteimport"
+
+
+class CodeType(enum.StrEnum):
+    source = "source"
+    bytecode = "bytecode"
+
+
+class Measurement(enum.StrEnum):
+    time = "time"
+    size = "size"
+
+
+PACKAGE_PATHS: dict[Importer, dict[CodeType, pathlib.Path]] = {
+    Importer.filesystem: {
+        CodeType.source: PACKAGE_DIRECTORY,
+        CodeType.bytecode: PACKAGE_DIRECTORY,
+    },
+    Importer.zipimport: {
+        CodeType.source: STATS / "source.zip",
+        CodeType.bytecode: STATS / "bytecode.zip",
+    },
+    Importer.sqliteimport: {
+        CodeType.source: STATS / "source.sqlite3",
+        CodeType.bytecode: STATS / "bytecode.sqlite3",
+    },
+}
+
+LOG_PATHS: dict[Importer, dict[CodeType, pathlib.Path]] = {
+    # This is a dictionary comprehension.
+    importer: {
+        code_type: STATS / f"{importer}.{code_type}.log" for code_type in CodeType
+    }
+    for importer in Importer
+}

assets/performance/__init__.py

@@ -0,0 +1,30 @@
+# This file is a part of sqliteimport <https://github.com/kurtmckee/sqliteimport>
+# Copyright 2024-2025 Kurt McKee <[email protected]>
+# SPDX-License-Identifier: MIT
+
+import pathlib
+import sys
+
+import click
+
+# Add this directory to the Python path so imports work.
+assets = pathlib.Path(__file__).parent.parent
+sys.path.insert(0, str(assets))
+
+
+if __name__ == "__main__":
+    from performance.clean import clean
+    from performance.collect import collect
+    from performance.compile import compile_
+    from performance.generate import generate
+    from performance.plot import plot
+    from performance.run import run
+
+    group = click.Group()
+    group.add_command(clean)
+    group.add_command(collect)
+    group.add_command(compile_)
+    group.add_command(generate)
+    group.add_command(plot)
+    group.add_command(run)
+    group()

assets/performance/__main__.py

@@ -0,0 +1,21 @@
+# This file is a part of sqliteimport <https://github.com/kurtmckee/sqliteimport>
+# Copyright 2024-2025 Kurt McKee <[email protected]>
+# SPDX-License-Identifier: MIT
+
+import shutil
+
+import click
+
+from . import PACKAGE_DIRECTORY
+from . import STATS
+
+
+@click.command()
+def clean() -> None:
+    """
+    Erase files and directories generated by performance testing script.
+    """
+
+    for path in (PACKAGE_DIRECTORY, STATS):
+        if path.is_dir():
+            shutil.rmtree(path)

assets/performance/clean.py

@@ -0,0 +1,128 @@
+# This file is a part of sqliteimport <https://github.com/kurtmckee/sqliteimport>
+# Copyright 2024-2025 Kurt McKee <[email protected]>
+# SPDX-License-Identifier: MIT
+
+import itertools
+import json
+import pathlib
+import typing
+
+import click
+
+from . import LOG_PATHS
+from . import PACKAGE_PATHS
+from . import STATS
+from . import CodeType
+from . import Importer
+from . import Measurement
+
+
+@click.command()
+def collect() -> None:
+    """
+    Collect import and sizing stats and write them to a JSON file.
+    """
+
+    stats = {
+        Measurement.time: parse_import_log(),
+        Measurement.size: get_size_stats(),
+    }
+    output_path = STATS / "stats.json"
+    output_path.parent.mkdir(exist_ok=True)
+    output_path.write_text(json.dumps(stats, indent=2, sort_keys=True))
+
+
+def parse_import_log() -> dict[str, dict[str, dict[str, int]]]:
+    """Parse an import log to extract per-module cumulative import times (in µs)."""
+
+    stats: dict[str, dict[str, dict[str, int]]] = {}
+    for importer, code_type in itertools.product(Importer, CodeType):
+        stats.setdefault(code_type, {})[importer] = {"-cumulative_us": 0}
+        file = LOG_PATHS[importer][code_type]
+        if not file.is_file():
+            click.echo(f"{file} not found")
+            continue
+
+        content = file.read_text()
+
+        for _, cumulative_us, module in split_columns(content):
+            if module.startswith(" "):
+                # The module is an indented submodule.
+                # Only top-level modules are recorded here.
+                continue
+            stats[code_type][importer][module] = cumulative_us
+            stats[code_type][importer]["-cumulative_us"] += cumulative_us
+
+    return stats
+
+
+def split_columns(text: str) -> typing.Iterator[tuple[int, int, str]]:
+    for line in text.splitlines():
+        prefix, _, remainder = line.partition(": ")
+        if prefix != "import time":
+            continue
+        try:
+            self, cumulative, module = remainder.split(" | ")
+            yield int(self.strip()), int(cumulative.strip()), module.rstrip()
+        except (TypeError, ValueError):
+            continue
+
+
+def get_size_stats() -> dict[Importer, dict[CodeType, int]]:
+    """Get the size of items on disk.
+
+    This only considers the raw sum of the sizes of files on disk.
+    It does not, for example, calculate the size of the filesystem blocks
+    that are consumed by individual files.
+    """
+
+    importer: Importer
+    code_type: CodeType
+
+    sizes: dict[Importer, dict[CodeType, int]] = {
+        importer: {code_type: -1 for code_type in CodeType} for importer in Importer
+    }
+    for importer, code_type in itertools.product(Importer, CodeType):
+        path = PACKAGE_PATHS[importer][code_type]
+        if path.is_dir():
+            sizes[importer][code_type] = get_directory_size(path, code_type)
+        elif path.is_file():
+            sizes[importer][code_type] = path.stat().st_size
+        else:
+            typing.assert_never((importer, code_type))
+
+    return sizes
+
+
+def get_directory_size(directory: pathlib.Path, code_type: CodeType) -> int:
+    """Get the size of a directory.
+
+    PEP3147-compatible ``*.pyc`` files will be included in the calculation
+    if the *code_type* indicates byte code.
+    """
+
+    size = 0
+
+    directories = [directory]
+    while directories:
+        directory = directories.pop()
+        for path in directory.glob("*"):
+            if path.is_dir():
+                # Skip `__pycache__/` directories unless calculating byte code size.
+                if path.name == "__pycache__" and code_type != CodeType.bytecode:
+                    continue
+                directories.append(path)
+
+            elif path.is_file():
+                # Skip `*.pyc` files unless calculating byte code size.
+                # Even then, the `*.pyc` files must be in a `__pycache__/` directory.
+                if path.suffix == ".pyc":
+                    if not (
+                        code_type == CodeType.bytecode
+                        and path.parent.name == "__pycache__"
+                    ):
+                        continue
+
+                size += path.stat().st_size
+
+    return size